pulling in DFs and Merging (still needed?)

# #red wines
# Red_wine <- read.csv("wineQualityReds.csv", header=TRUE, sep = ",")
# Red_wine$Type <- 'Red'
# 
# #white wines
# White_wine <- read.csv("wineQualityWhites.csv", header=TRUE, sep = ",")
# White_wine$Type <- 'White'
# 
# ## consolidated
# Data <- rbind(Red_wine,White_wine)
# drops <- c("X")
# Data <- Data[ , !(names(Data) %in% drops)]
# Data
# 
# ##create final DF
# # write.csv(Data,"/Users/colinobrien/Desktop/repo/stats_6021/Stats_project_group_6/Data.csv", row.names = FALSE)
# # write.csv(Data,"/Users/colinobrien/Desktop/repo/stats_6021/Stats_project_group_6/Data", row.names = FALSE)
# ## both the Data and Data csv are the same. I know people prefer one format vs the other so I made both
library(tidyverse)
Registered S3 methods overwritten by 'dbplyr':
  method         from
  print.tbl_lazy     
  print.tbl_sql      
── Attaching packages ────────────────────────────────────────────────────────────── tidyverse 1.3.1 ──
✓ ggplot2 3.3.5     ✓ purrr   0.3.4
✓ tibble  3.1.2     ✓ dplyr   1.0.7
✓ tidyr   1.1.3     ✓ stringr 1.4.0
✓ readr   1.4.0     ✓ forcats 0.5.1
── Conflicts ───────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
x dplyr::filter() masks stats::filter()
x dplyr::lag()    masks stats::lag()
# library(ROCR)
library(faraway)
library(dplyr)
library(ggplot2)
library(reshape2)

Attaching package: ‘reshape2’

The following object is masked from ‘package:tidyr’:

    smiths
library(leaps)
# install.packages("bestglm")
library(bestglm)
# install.packages("performance")
# library(performance)
knitr::opts_chunk$set(echo = TRUE)



## Load Datasets
full_wines_final <- read.csv("Data_Final.csv", header = TRUE, stringsAsFactors=TRUE)
# Drop quality for simplicity
full_wines_binary_with_qual<-full_wines_final
full_wines_binary <- subset(full_wines_final, select = -c(quality))
## Convert to 0 and 1 for readability
full_wines_binary$cat_quality <- as.integer(full_wines_binary$cat_quality == "High")

set.seed(90210) ##for reproducibility
sample<-sample.int(nrow(full_wines_binary), floor(.80*nrow(full_wines_binary)), replace = F)
train<-full_wines_binary[sample, ] ##training data frame
rownames(train) <- c(1:5197)
test<-full_wines_binary[-sample, ] ##test data frame

## Just for a single boxplot
train_with_qual<-full_wines_binary_with_qual[sample,]
test_with_qual<-full_wines_binary_with_qual[-sample,]


train

EDA

# drops_cats <- c("Type")
# No_cat_train <- train[ , !(names(train) %in% drops_cats)]
# # No_Type

pairs(train, lower.panel = NULL)

# Convert Type to binary to 0 and 1 for correlation
train$Type <- as.integer(train$Type == "White")
test$Type <- as.integer(test$Type == "White")
cor_train <- cor(train)
cor_train
                     fixed.acidity volatile.acidity   citric.acid residual.sugar   chlorides
fixed.acidity           1.00000000       0.21475470  0.3281717780    -0.11435784  0.30907486
volatile.acidity        0.21475470       1.00000000 -0.3756083338    -0.19711519  0.38572698
citric.acid             0.32817178      -0.37560833  1.0000000000     0.14135328  0.03681626
residual.sugar         -0.11435784      -0.19711519  0.1413532761     1.00000000 -0.13267263
chlorides               0.30907486       0.38572698  0.0368162603    -0.13267263  1.00000000
free.sulfur.dioxide    -0.28545891      -0.36296634  0.1459151441     0.40674512 -0.20583891
total.sulfur.dioxide   -0.32829667      -0.42380894  0.2063508767     0.49459459 -0.29216595
density                 0.46102677       0.27340816  0.0934734894     0.54686423  0.36564000
pH                     -0.25053297       0.26430327 -0.3266522997    -0.26561668  0.04191555
sulphates               0.31025439       0.23565770  0.0572809414    -0.18182649  0.40162365
alcohol                -0.09179407      -0.03401873 -0.0006471143    -0.34842765 -0.25617404
Type                   -0.48713134      -0.65599571  0.1886904967     0.35126769 -0.52157700
cat_quality            -0.07066799      -0.26538562  0.0778223488    -0.02120537 -0.18458731
                     free.sulfur.dioxide total.sulfur.dioxide     density          pH   sulphates
fixed.acidity                -0.28545891          -0.32829667  0.46102677 -0.25053297  0.31025439
volatile.acidity             -0.36296634          -0.42380894  0.27340816  0.26430327  0.23565770
citric.acid                   0.14591514           0.20635088  0.09347349 -0.32665230  0.05728094
residual.sugar                0.40674512           0.49459459  0.54686423 -0.26561668 -0.18182649
chlorides                    -0.20583891          -0.29216595  0.36564000  0.04191555  0.40162365
free.sulfur.dioxide           1.00000000           0.71557904  0.01712365 -0.15788076 -0.19860411
total.sulfur.dioxide          0.71557904           1.00000000  0.02373810 -0.24648617 -0.27752789
density                       0.01712365           0.02373810  1.00000000  0.01687243  0.27270901
pH                           -0.15788076          -0.24648617  0.01687243  1.00000000  0.18260830
sulphates                    -0.19860411          -0.27752789  0.27270901  0.18260830  1.00000000
alcohol                      -0.17708863          -0.26082628 -0.67847127  0.11700658 -0.01360657
Type                          0.48355395           0.70618430 -0.39538819 -0.33097301 -0.49215645
cat_quality                   0.04477089          -0.04308093 -0.26437860  0.01854094  0.03195034
                           alcohol        Type cat_quality
fixed.acidity        -0.0917940736 -0.48713134 -0.07066799
volatile.acidity     -0.0340187335 -0.65599571 -0.26538562
citric.acid          -0.0006471143  0.18869050  0.07782235
residual.sugar       -0.3484276488  0.35126769 -0.02120537
chlorides            -0.2561740351 -0.52157700 -0.18458731
free.sulfur.dioxide  -0.1770886318  0.48355395  0.04477089
total.sulfur.dioxide -0.2608262814  0.70618430 -0.04308093
density              -0.6784712690 -0.39538819 -0.26437860
pH                    0.1170065815 -0.33097301  0.01854094
sulphates            -0.0136065691 -0.49215645  0.03195034
alcohol               1.0000000000  0.03945167  0.39668183
Type                  0.0394516700  1.00000000  0.12361294
cat_quality           0.3966818275  0.12361294  1.00000000
T_F_cor <- abs(cor_train)>.7
T_F_cor
                     fixed.acidity volatile.acidity citric.acid residual.sugar chlorides
fixed.acidity                 TRUE            FALSE       FALSE          FALSE     FALSE
volatile.acidity             FALSE             TRUE       FALSE          FALSE     FALSE
citric.acid                  FALSE            FALSE        TRUE          FALSE     FALSE
residual.sugar               FALSE            FALSE       FALSE           TRUE     FALSE
chlorides                    FALSE            FALSE       FALSE          FALSE      TRUE
free.sulfur.dioxide          FALSE            FALSE       FALSE          FALSE     FALSE
total.sulfur.dioxide         FALSE            FALSE       FALSE          FALSE     FALSE
density                      FALSE            FALSE       FALSE          FALSE     FALSE
pH                           FALSE            FALSE       FALSE          FALSE     FALSE
sulphates                    FALSE            FALSE       FALSE          FALSE     FALSE
alcohol                      FALSE            FALSE       FALSE          FALSE     FALSE
Type                         FALSE            FALSE       FALSE          FALSE     FALSE
cat_quality                  FALSE            FALSE       FALSE          FALSE     FALSE
                     free.sulfur.dioxide total.sulfur.dioxide density    pH sulphates alcohol  Type
fixed.acidity                      FALSE                FALSE   FALSE FALSE     FALSE   FALSE FALSE
volatile.acidity                   FALSE                FALSE   FALSE FALSE     FALSE   FALSE FALSE
citric.acid                        FALSE                FALSE   FALSE FALSE     FALSE   FALSE FALSE
residual.sugar                     FALSE                FALSE   FALSE FALSE     FALSE   FALSE FALSE
chlorides                          FALSE                FALSE   FALSE FALSE     FALSE   FALSE FALSE
free.sulfur.dioxide                 TRUE                 TRUE   FALSE FALSE     FALSE   FALSE FALSE
total.sulfur.dioxide                TRUE                 TRUE   FALSE FALSE     FALSE   FALSE  TRUE
density                            FALSE                FALSE    TRUE FALSE     FALSE   FALSE FALSE
pH                                 FALSE                FALSE   FALSE  TRUE     FALSE   FALSE FALSE
sulphates                          FALSE                FALSE   FALSE FALSE      TRUE   FALSE FALSE
alcohol                            FALSE                FALSE   FALSE FALSE     FALSE    TRUE FALSE
Type                               FALSE                 TRUE   FALSE FALSE     FALSE   FALSE  TRUE
cat_quality                        FALSE                FALSE   FALSE FALSE     FALSE   FALSE FALSE
                     cat_quality
fixed.acidity              FALSE
volatile.acidity           FALSE
citric.acid                FALSE
residual.sugar             FALSE
chlorides                  FALSE
free.sulfur.dioxide        FALSE
total.sulfur.dioxide       FALSE
density                    FALSE
pH                         FALSE
sulphates                  FALSE
alcohol                    FALSE
Type                       FALSE
cat_quality                 TRUE
## create melted
melted_cor_train <- melt(cor_train)

##create heat map Consolidated
ggplot(data = melted_cor_train, aes(x=Var1, y=Var2, fill=value)) + 
  geom_tile(color = "white")+
  scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
  theme_minimal()+ 
  theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
  coord_fixed()+
  labs(title = 'Consolidated (Both Red and White)')

NA
NA
NA
NA
## creating red and white
train_White <- filter(train, Type == 1)
train_Red <- filter(train, Type == 0)


## droping red/white columns
train_White_NoType <- subset(train_White, select = -c(Type))
train_Red_NoType <- subset(train_Red, select = -c(Type))

## creating correlations
cor_train_White_NoType <- cor(train_White_NoType)
cor_train_Red_NoType <- cor(train_Red_NoType)

## melting
melted_cor_train_white <- melt(cor_train_White_NoType)
melted_cor_train_Red <- melt(cor_train_Red_NoType)

##ploting

##create heat map White
ggplot(data = melted_cor_train_white, aes(x=Var1, y=Var2, fill=value)) +
  geom_tile(color = "white")+
  scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
  theme_minimal()+
  theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
  coord_fixed()+
  labs(title = 'White Wine')


##create heat map Red
ggplot(data = melted_cor_train_Red, aes(x=Var1, y=Var2, fill=value)) +
  geom_tile(color = "white")+
  scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
  theme_minimal()+
  theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
  coord_fixed()+
  labs(title = 'Red Wine')

ggplot(data = train, mapping = aes(x=Type)) + 
  geom_bar()

Regression Testing

## press formula (from class)
get_press <- function(model) {
  sum(((model$residuals)/ (1- (lm.influence(model)$hat)))^2)
}
## first go
full<-glm(cat_quality~., family=binomial, data=train)
summary(full)

Call:
glm(formula = cat_quality ~ ., family = binomial, data = train)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.4120  -0.8919   0.4303   0.8148   2.6198  

Coefficients:
                       Estimate Std. Error z value Pr(>|z|)    
(Intercept)           1.102e+02  4.889e+01   2.253   0.0242 *  
fixed.acidity         7.676e-02  5.573e-02   1.377   0.1684    
volatile.acidity     -4.720e+00  3.291e-01 -14.342  < 2e-16 ***
citric.acid          -4.652e-01  2.875e-01  -1.618   0.1057    
residual.sugar        1.187e-01  2.104e-02   5.641 1.69e-08 ***
chlorides            -1.265e+00  1.195e+00  -1.059   0.2895    
free.sulfur.dioxide   1.345e-02  2.878e-03   4.672 2.98e-06 ***
total.sulfur.dioxide -5.753e-03  1.168e-03  -4.924 8.48e-07 ***
density              -1.211e+02  4.964e+01  -2.440   0.0147 *  
pH                    7.375e-01  3.316e-01   2.224   0.0261 *  
sulphates             2.096e+00  2.977e-01   7.042 1.89e-12 ***
alcohol               8.500e-01  6.635e-02  12.811  < 2e-16 ***
Type                 -5.562e-01  2.077e-01  -2.678   0.0074 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 6833.2  on 5196  degrees of freedom
Residual deviance: 5335.5  on 5184  degrees of freedom
AIC: 5361.5

Number of Fisher Scoring iterations: 5
## removed all insignificant
reduced_1<-glm(formula = cat_quality~volatile.acidity+residual.sugar+free.sulfur.dioxide+total.sulfur.dioxide+density+pH+sulphates+alcohol+Type, family=binomial, data=train)
summary(reduced_1)

Call:
glm(formula = cat_quality ~ volatile.acidity + residual.sugar + 
    free.sulfur.dioxide + total.sulfur.dioxide + density + pH + 
    sulphates + alcohol + Type, family = binomial, data = train)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.3471  -0.9020   0.4275   0.8196   2.6625  

Coefficients:
                       Estimate Std. Error z value Pr(>|z|)    
(Intercept)           76.416878  31.773373   2.405  0.01617 *  
volatile.acidity      -4.680054   0.305603 -15.314  < 2e-16 ***
residual.sugar         0.105875   0.014724   7.191 6.44e-13 ***
free.sulfur.dioxide    0.013183   0.002863   4.604 4.15e-06 ***
total.sulfur.dioxide  -0.005930   0.001162  -5.101 3.38e-07 ***
density              -86.733244  31.675484  -2.738  0.00618 ** 
pH                     0.595611   0.228953   2.601  0.00928 ** 
sulphates              1.930494   0.285976   6.751 1.47e-11 ***
alcohol                0.894713   0.051370  17.417  < 2e-16 ***
Type                  -0.523798   0.204855  -2.557  0.01056 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 6833.2  on 5196  degrees of freedom
Residual deviance: 5341.5  on 5187  degrees of freedom
AIC: 5361.5

Number of Fisher Scoring iterations: 5
##evaluating model
Reduced1_AIC_train <- reduced_1$aic

##predicted quality for test data based on training data
preds<-predict(reduced_1,newdata=test, type="response")

reduced_1_error <- table(test$cat_quality, preds>0.5)

reduced_1_error
   
    FALSE TRUE
  0   240  236
  1   121  703
evulation_summary <- data.frame(
  attempt = 'reduced_1',
  AIC = Reduced1_AIC_train,
  PRESS = get_press(reduced_1),
  'False positive' = round(reduced_1_error[3]/(reduced_1_error[1]+reduced_1_error[3]),3),
  'False negative' = round(reduced_1_error[2]/(reduced_1_error[2]+reduced_1_error[4]),3),
  'Error Rate' = round((reduced_1_error[2]+reduced_1_error[3])/(reduced_1_error[1]+reduced_1_error[2]+reduced_1_error[3]+reduced_1_error[4]),3)
)
evulation_summary

second model

https://rstudio-pubs-static.s3.amazonaws.com/2897_9220b21cfc0c43a396ff9abf122bb351.html

# install.packages("bestglm")
## Prepare data
train.for.best.logistic <- within(train, {
    y <- cat_quality 
})

## Reorder variables
train.for.best.logistic <-
    train.for.best.logistic[, c("fixed.acidity","volatile.acidity","citric.acid","residual.sugar","total.sulfur.dioxide","density","chlorides","free.sulfur.dioxide",'pH','sulphates','alcohol','Type',"y")]

## Perform
res.best.logistic <-
    bestglm(Xy = train.for.best.logistic,
            family = binomial,          # binomial family for logistic
            IC = "AIC",                 # Information criteria for
            method = "exhaustive")
Morgan-Tatar search since family is non-gaussian.
res.best.logistic$BestModels
summary(res.best.logistic$BestModel)

Call:
glm(formula = y ~ ., family = family, data = Xi, weights = weights)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.4302  -0.8932   0.4282   0.8157   2.6350  

Coefficients:
                       Estimate Std. Error z value Pr(>|z|)    
(Intercept)           1.147e+02  4.864e+01   2.358   0.0184 *  
fixed.acidity         8.518e-02  5.513e-02   1.545   0.1223    
volatile.acidity     -4.763e+00  3.268e-01 -14.574  < 2e-16 ***
citric.acid          -5.158e-01  2.834e-01  -1.820   0.0688 .  
residual.sugar        1.215e-01  2.085e-02   5.829 5.57e-09 ***
total.sulfur.dioxide -5.688e-03  1.167e-03  -4.876 1.08e-06 ***
density              -1.261e+02  4.936e+01  -2.554   0.0106 *  
free.sulfur.dioxide   1.332e-02  2.874e-03   4.633 3.61e-06 ***
pH                    8.012e-01  3.261e-01   2.457   0.0140 *  
sulphates             2.027e+00  2.897e-01   6.998 2.59e-12 ***
alcohol               8.553e-01  6.613e-02  12.934  < 2e-16 ***
Type                 -5.290e-01  2.059e-01  -2.569   0.0102 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 6833.2  on 5196  degrees of freedom
Residual deviance: 5336.7  on 5185  degrees of freedom
AIC: 5360.7

Number of Fisher Scoring iterations: 5
reduced_4 <- res.best.logistic$BestModel
##evaluating model
Reduced4_AIC_train <- reduced_4$aic

##predicted quality for test data based on training data
preds<-predict(reduced_4,newdata=test, type="response")

reduced_4_error <- table(test$cat_quality, preds>0.5)

evulation_summary_4 <- data.frame(
  attempt = 'reduced_4_error (all possible)',
  AIC = Reduced4_AIC_train,
  PRESS = get_press(reduced_4),
  'False positive' = round(reduced_4_error[3]/(reduced_4_error[1]+reduced_4_error[3]),3),
  'False negative' = round(reduced_4_error[2]/(reduced_4_error[2]+reduced_4_error[4]),3),
  'Error Rate' = round((reduced_4_error[2]+reduced_4_error[3])/(reduced_4_error[1]+reduced_4_error[2]+reduced_4_error[3]+reduced_4_error[4]),3)
)

evulation_summary <- rbind(evulation_summary,evulation_summary_4)
# evulation_summary
# data.frame(check_collinearity(reduced_4))










#come back and add df stuff

in an effort to lower VIFs scores and correlation, I am removing fixed.acidity

reduced_4_2<-glm(cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type, family=binomial, data=train)
summary(reduced_4_2)

Call:
glm(formula = cat_quality ~ volatile.acidity + citric.acid + 
    residual.sugar + total.sulfur.dioxide + density + free.sulfur.dioxide + 
    pH + sulphates + alcohol + Type, family = binomial, data = train)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.3469  -0.8997   0.4283   0.8144   2.6767  

Coefficients:
                       Estimate Std. Error z value Pr(>|z|)    
(Intercept)           58.609496  33.904591   1.729   0.0839 .  
volatile.acidity      -4.841743   0.323737 -14.956  < 2e-16 ***
citric.acid           -0.434875   0.278458  -1.562   0.1184    
residual.sugar         0.099352   0.015302   6.493 8.43e-11 ***
total.sulfur.dioxide  -0.005815   0.001165  -4.993 5.94e-07 ***
density              -68.496442  33.901654  -2.020   0.0433 *  
free.sulfur.dioxide    0.013292   0.002871   4.630 3.66e-06 ***
pH                     0.465949   0.243355   1.915   0.0555 .  
sulphates              1.962718   0.287251   6.833 8.33e-12 ***
alcohol                0.918353   0.053777  17.077  < 2e-16 ***
Type                  -0.483884   0.206617  -2.342   0.0192 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 6833.2  on 5196  degrees of freedom
Residual deviance: 5339.0  on 5186  degrees of freedom
AIC: 5361

Number of Fisher Scoring iterations: 5
##evaluating model
Reduced4_2_AIC_train <- reduced_4_2$aic
##predicted quality for test data based on training data
preds<-predict(reduced_4_2,newdata=test, type="response")
reduced_4_2_error <- table(test$cat_quality, preds>0.7)
#Curves
evulation_summary_4_2 <- data.frame(
  attempt = 'reduced_4_2_error (post VIF adjustments)',
  AIC = Reduced4_2_AIC_train,
  PRESS = get_press(reduced_4_2),
  'False positive' = round(reduced_4_2_error[3]/(reduced_4_2_error[1]+reduced_4_2_error[3]),3),
  'False negative' = round(reduced_4_2_error[2]/(reduced_4_2_error[2]+reduced_4_2_error[4]),3),
  'Error Rate' = round((reduced_4_2_error[2]+reduced_4_2_error[3])/(reduced_4_2_error[1]+reduced_4_2_error[2]+reduced_4_2_error[3]+reduced_4_2_error[4]),3)
)

evulation_summary <- rbind(evulation_summary,evulation_summary_4_2)
evulation_summary
NA
NA

now looking at outliers (with “best possible”)

summary(reduced_4)

Call:
glm(formula = y ~ ., family = family, data = Xi, weights = weights)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.4302  -0.8932   0.4282   0.8157   2.6350  

Coefficients:
                       Estimate Std. Error z value Pr(>|z|)    
(Intercept)           1.147e+02  4.864e+01   2.358   0.0184 *  
fixed.acidity         8.518e-02  5.513e-02   1.545   0.1223    
volatile.acidity     -4.763e+00  3.268e-01 -14.574  < 2e-16 ***
citric.acid          -5.158e-01  2.834e-01  -1.820   0.0688 .  
residual.sugar        1.215e-01  2.085e-02   5.829 5.57e-09 ***
total.sulfur.dioxide -5.688e-03  1.167e-03  -4.876 1.08e-06 ***
density              -1.261e+02  4.936e+01  -2.554   0.0106 *  
free.sulfur.dioxide   1.332e-02  2.874e-03   4.633 3.61e-06 ***
pH                    8.012e-01  3.261e-01   2.457   0.0140 *  
sulphates             2.027e+00  2.897e-01   6.998 2.59e-12 ***
alcohol               8.553e-01  6.613e-02  12.934  < 2e-16 ***
Type                 -5.290e-01  2.059e-01  -2.569   0.0102 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 6833.2  on 5196  degrees of freedom
Residual deviance: 5336.7  on 5185  degrees of freedom
AIC: 5360.7

Number of Fisher Scoring iterations: 5

Now looking into outliers/influence

p <- 12
n <- 5197

Cooks

reduced_4_cook <-cooks.distance(reduced_4)
reduced_4_cook[reduced_4_cook>qf(0.5,p,n-p)]
named numeric(0)

DFFITs

##dffits
DFFITS<-dffits(reduced_4)
DDFFITS_influence <- DFFITS[abs(DFFITS)>2*sqrt(p/n)]
DDFFITS_influence
         83         115         133         148         149         171         183         202 
 0.11219627  0.10544184 -0.18411002 -0.09991644 -0.12785114 -0.11374142  0.10867107 -0.16045065 
        206         225         249         292         309         317         380         455 
-0.11840913  0.10093853 -0.09841249  0.09816267  0.10078385  0.10471012 -0.11210579  0.09920619 
        466         572         722         739         747         794         806         912 
-0.11390124  0.10171119 -0.16019151  0.11513457  0.09736192 -0.10449640 -0.10120073  0.15167391 
        952         957        1010        1042        1055        1114        1139        1180 
 0.09704330 -0.16243046  0.11892944 -0.09863713 -0.11344570 -0.10497624  0.10300372 -0.18433803 
       1241        1258        1270        1325        1347        1467        1493        1527 
 0.10969510  0.09821539  0.10530848 -0.09678052 -0.10103881  0.11642941 -0.11091048 -0.11561143 
       1528        1597        1620        1658        1693        1776        1846        1873 
-0.12352731 -0.09863589 -0.10722071 -0.09731307 -0.09795498 -0.10237941  0.12561590 -0.12529594 
       1882        1891        2002        2004        2020        2079        2103        2148 
-0.10642853  0.10446926  0.10894528  0.20125104  0.09637786 -0.10426201  0.09853506 -0.27921127 
       2203        2256        2258        2285        2305        2350        2353        2361 
-0.10129823 -0.10012511  0.10312207  0.10896369  0.13774801  0.09853162 -0.15221240 -0.11012001 
       2383        2410        2434        2461        2465        2487        2494        2497 
-0.10852123 -0.12636270  0.12603788 -0.15438799 -0.10048416  0.10125735 -0.12151311  0.10319545 
       2521        2525        2527        2551        2594        2650        2696        2699 
-0.10237941 -0.10131647 -0.10242317 -0.09669244 -0.10722071 -0.10178224 -0.11840913 -0.10221624 
       2772        2781        2801        2840        2852        2892        2948        2965 
 0.20008361 -0.10703304 -0.11210579  0.16042062 -0.10974330  0.09732574 -0.09841249 -0.09647583 
       2978        2994        2995        2997        3018        3030        3041        3051 
-0.11806742  0.14196649  0.09920619 -0.13745390  0.10099437 -0.14516539  0.16695970 -0.12151311 
       3065        3079        3126        3155        3188        3274        3282        3330 
-0.21978199  0.10160629  0.17165718  0.13048253  0.17504970 -0.10882370  0.10171119  0.09880694 
       3347        3367        3378        3381        3496        3526        3557        3564 
-0.11538312 -0.11374142 -0.10444247  0.10177520  0.12512066 -0.15993715 -0.13072803 -0.13765634 
       3571        3709        3721        3740        3752        3809        3823        3843 
-0.11317090 -0.11344570 -0.15438799 -0.10404207 -0.15579966 -0.10786452  0.88209659  0.10113003 
       3848        3895        3922        4024        4035        4065        4071        4112 
 0.11006220 -0.18174866 -0.12028121 -0.10640867 -0.11091048  0.09637786 -0.31948427 -0.12024521 
       4125        4131        4132        4140        4260        4300        4401        4425 
-0.10057784 -0.10147570  0.17165718 -0.11174867 -0.11267738  0.10499625 -0.10880237 -0.10594437 
       4463        4490        4512        4547        4610        4627        4678        4687 
-0.12634536 -0.09991644 -0.09647583 -0.17220454  0.10741187 -0.21859387 -0.10392610  0.12149435 
       4783        4790        4794        4829        4873        5045        5091        5178 
 0.12315874  0.10969510 -0.09878535 -0.11104591 -0.16746164 -0.10529190 -0.10703304  0.11264144 
       5187 
 0.10894528 

DFBETAs

DFBETAS<-dfbetas(reduced_4)
abs(DFBETAS)>2/sqrt(n)
     (Intercept) fixed.acidity volatile.acidity citric.acid residual.sugar total.sulfur.dioxide
1          FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
2          FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
3          FALSE          TRUE            FALSE        TRUE           TRUE                FALSE
4          FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
5          FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
6          FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
7          FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
8          FALSE         FALSE            FALSE       FALSE           TRUE                FALSE
9          FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
10          TRUE          TRUE            FALSE        TRUE           TRUE                 TRUE
11         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
12         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
13         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
14         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
15         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
16         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
17         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
18         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
19         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
20         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
21         FALSE         FALSE            FALSE        TRUE          FALSE                FALSE
22         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
23         FALSE          TRUE            FALSE       FALSE           TRUE                FALSE
24         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
25         FALSE          TRUE             TRUE       FALSE          FALSE                FALSE
26         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
27         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
28         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
29         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
30         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
31         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
32         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
33         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
34         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
35         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
36         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
37         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
38         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
39         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
40         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
41         FALSE         FALSE             TRUE       FALSE          FALSE                FALSE
42         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
43         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
44         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
45         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
46         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
47         FALSE         FALSE            FALSE       FALSE           TRUE                FALSE
48         FALSE         FALSE            FALSE        TRUE          FALSE                FALSE
49         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
50         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
51         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
52         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
53         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
54         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
55         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
56         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
57         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
58         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
59         FALSE         FALSE             TRUE        TRUE          FALSE                FALSE
60         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
61         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
62         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
63         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
64         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
65          TRUE          TRUE            FALSE       FALSE           TRUE                FALSE
66         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
67         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
68         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
69         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
70         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
71         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
72         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
73         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
74         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
75         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
76         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
77         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
78         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
79         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
80         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
81         FALSE          TRUE            FALSE       FALSE          FALSE                FALSE
82         FALSE         FALSE            FALSE       FALSE          FALSE                FALSE
83         FALSE          TRUE             TRUE        TRUE          FALSE                FALSE
     density free.sulfur.dioxide    pH sulphates alcohol  Type
1      FALSE               FALSE FALSE     FALSE   FALSE FALSE
2      FALSE               FALSE FALSE     FALSE   FALSE FALSE
3      FALSE               FALSE  TRUE     FALSE   FALSE FALSE
4      FALSE               FALSE FALSE     FALSE   FALSE FALSE
5      FALSE               FALSE FALSE     FALSE   FALSE FALSE
6      FALSE               FALSE FALSE     FALSE   FALSE  TRUE
7      FALSE               FALSE FALSE     FALSE   FALSE FALSE
8      FALSE                TRUE FALSE      TRUE   FALSE FALSE
9      FALSE               FALSE FALSE     FALSE   FALSE FALSE
10      TRUE                TRUE FALSE     FALSE    TRUE  TRUE
11     FALSE               FALSE FALSE     FALSE   FALSE FALSE
12     FALSE               FALSE FALSE     FALSE   FALSE FALSE
13     FALSE               FALSE FALSE     FALSE   FALSE FALSE
14     FALSE               FALSE FALSE     FALSE   FALSE FALSE
15     FALSE               FALSE FALSE      TRUE   FALSE FALSE
16     FALSE               FALSE FALSE     FALSE   FALSE FALSE
17     FALSE               FALSE FALSE      TRUE   FALSE FALSE
18     FALSE               FALSE  TRUE     FALSE   FALSE FALSE
19     FALSE               FALSE FALSE     FALSE   FALSE FALSE
20     FALSE               FALSE FALSE     FALSE   FALSE FALSE
21     FALSE               FALSE FALSE     FALSE   FALSE FALSE
22     FALSE               FALSE FALSE     FALSE   FALSE FALSE
23     FALSE               FALSE FALSE     FALSE   FALSE FALSE
24     FALSE               FALSE FALSE     FALSE   FALSE FALSE
25     FALSE               FALSE FALSE     FALSE   FALSE FALSE
26     FALSE               FALSE FALSE     FALSE   FALSE FALSE
27     FALSE               FALSE FALSE     FALSE   FALSE FALSE
28     FALSE               FALSE FALSE     FALSE   FALSE FALSE
29     FALSE               FALSE FALSE     FALSE   FALSE FALSE
30     FALSE               FALSE FALSE     FALSE   FALSE FALSE
31     FALSE               FALSE FALSE     FALSE   FALSE FALSE
32     FALSE               FALSE FALSE     FALSE   FALSE FALSE
33     FALSE               FALSE FALSE     FALSE   FALSE FALSE
34     FALSE               FALSE FALSE     FALSE   FALSE FALSE
35     FALSE               FALSE FALSE     FALSE   FALSE FALSE
36     FALSE               FALSE FALSE     FALSE   FALSE FALSE
37     FALSE               FALSE FALSE     FALSE   FALSE FALSE
38     FALSE               FALSE FALSE     FALSE   FALSE FALSE
39     FALSE               FALSE FALSE     FALSE   FALSE FALSE
40     FALSE               FALSE FALSE     FALSE   FALSE FALSE
41     FALSE               FALSE FALSE      TRUE   FALSE  TRUE
42     FALSE               FALSE FALSE     FALSE   FALSE FALSE
43     FALSE                TRUE FALSE      TRUE   FALSE FALSE
44     FALSE               FALSE FALSE     FALSE   FALSE FALSE
45     FALSE               FALSE FALSE     FALSE   FALSE FALSE
46     FALSE               FALSE FALSE     FALSE   FALSE FALSE
47     FALSE               FALSE  TRUE     FALSE   FALSE FALSE
48     FALSE               FALSE FALSE     FALSE   FALSE FALSE
49     FALSE               FALSE FALSE     FALSE   FALSE FALSE
50     FALSE               FALSE FALSE     FALSE   FALSE FALSE
51     FALSE               FALSE FALSE     FALSE   FALSE FALSE
52     FALSE               FALSE FALSE     FALSE   FALSE FALSE
53     FALSE               FALSE FALSE     FALSE   FALSE FALSE
54     FALSE               FALSE FALSE     FALSE   FALSE FALSE
55     FALSE               FALSE FALSE     FALSE   FALSE FALSE
56     FALSE               FALSE FALSE     FALSE   FALSE FALSE
57     FALSE               FALSE FALSE     FALSE   FALSE FALSE
58     FALSE               FALSE FALSE     FALSE   FALSE FALSE
59     FALSE               FALSE FALSE     FALSE   FALSE  TRUE
60     FALSE               FALSE FALSE     FALSE   FALSE FALSE
61     FALSE               FALSE  TRUE     FALSE   FALSE FALSE
62     FALSE               FALSE FALSE     FALSE   FALSE FALSE
63     FALSE               FALSE FALSE     FALSE   FALSE FALSE
64     FALSE               FALSE FALSE     FALSE   FALSE FALSE
65      TRUE               FALSE  TRUE     FALSE    TRUE FALSE
66     FALSE               FALSE FALSE     FALSE   FALSE FALSE
67     FALSE               FALSE FALSE     FALSE   FALSE FALSE
68     FALSE               FALSE FALSE     FALSE   FALSE FALSE
69     FALSE               FALSE FALSE     FALSE   FALSE FALSE
70     FALSE                TRUE FALSE     FALSE   FALSE FALSE
71     FALSE               FALSE FALSE     FALSE   FALSE FALSE
72     FALSE               FALSE FALSE     FALSE   FALSE FALSE
73     FALSE               FALSE FALSE     FALSE   FALSE FALSE
74     FALSE               FALSE FALSE     FALSE   FALSE FALSE
75     FALSE               FALSE FALSE     FALSE   FALSE FALSE
76     FALSE               FALSE FALSE     FALSE   FALSE FALSE
77     FALSE               FALSE FALSE     FALSE   FALSE FALSE
78     FALSE               FALSE FALSE     FALSE   FALSE FALSE
79     FALSE               FALSE FALSE     FALSE   FALSE FALSE
80     FALSE               FALSE FALSE     FALSE   FALSE FALSE
81     FALSE               FALSE  TRUE     FALSE   FALSE FALSE
82     FALSE               FALSE FALSE     FALSE   FALSE FALSE
83     FALSE               FALSE  TRUE     FALSE   FALSE  TRUE
 [ reached getOption("max.print") -- omitted 5114 rows ]

leverage

##leverages
lev<-lm.influence(reduced_4)$hat
##identify high leverage points
leverages <- lev[lev>2*p/n]
leverages
         25          48          53          65         133         145         171         175 
0.004849640 0.007980011 0.007028687 0.005763404 0.022975591 0.005173793 0.005219474 0.004875721 
        187         202         225         228         248         249         255         282 
0.006161515 0.005447858 0.006756723 0.005384983 0.008357256 0.004801962 0.005867337 0.005674515 
        299         309         332         333         339         345         361         380 
0.005484537 0.010250038 0.004754212 0.004686940 0.004990699 0.005924282 0.006389325 0.006435227 
        383         417         427         455         466         489         516         557 
0.006875256 0.004943798 0.004867913 0.009165489 0.006894948 0.007172970 0.010652503 0.005554653 
        600         607         708         722         730         747         749         765 
0.005521417 0.005900775 0.004918820 0.005283197 0.005147385 0.005767084 0.005739296 0.005627857 
        770         806         828         873         899         912         945         957 
0.004853531 0.005540805 0.007828147 0.006995196 0.005410461 0.009841709 0.005463074 0.011974020 
        965         995        1010        1037        1055        1114        1132        1139 
0.005623028 0.014123202 0.004921736 0.006212390 0.006141910 0.005265677 0.006966068 0.005959266 
       1140        1152        1169        1176        1180        1188        1241        1255 
0.006430522 0.006059376 0.004777701 0.005844926 0.012489770 0.004953075 0.004856461 0.004777701 
       1265        1300        1304        1306        1325        1345        1375        1427 
0.005672314 0.006081786 0.005100154 0.005735362 0.005574241 0.004711974 0.006383030 0.004930082 
       1467        1479        1481        1482        1493        1495        1507        1518 
0.006002690 0.006720302 0.005242150 0.014123202 0.007969759 0.004698112 0.004854829 0.005124711 
       1527        1528        1612        1614        1638        1642        1649        1667 
0.005382525 0.005299431 0.009822894 0.008993399 0.007179770 0.007952008 0.005823824 0.007088896 
       1711        1719        1768        1801        1829        1845        1846        1873 
0.004814463 0.004889259 0.004851478 0.004848096 0.004766974 0.006026022 0.005410441 0.005408285 
       1882        1887        1891        1894        1929        1933        1935        1941 
0.004657424 0.005524176 0.005158734 0.004960904 0.007033838 0.006340479 0.005367322 0.005155488 
       1956        1979        2002        2004        2019        2097        2106        2120 
0.004781888 0.006023785 0.005565063 0.013293498 0.005674515 0.004945639 0.005391888 0.005466349 
       2135        2137        2144        2148        2155        2165        2177        2203 
0.006032961 0.005458302 0.005593131 0.020382883 0.005008204 0.005477897 0.005419422 0.004636068 
       2211        2218        2232        2270        2281        2294        2305        2306 
0.004669295 0.019421769 0.006338349 0.007220018 0.006915245 0.006786840 0.010603689 0.005150988 
       2317        2350        2353        2361        2371        2379        2383        2391 
0.011839638 0.004828941 0.012864370 0.005000134 0.005408758 0.005477511 0.006524792 0.005514256 
       2396        2400        2410        2420        2434        2461        2469        2494 
0.008839432 0.007088896 0.007468644 0.004680633 0.020272029 0.016139547 0.005146048 0.008984307 
       2497        2525        2602        2623        2650        2663        2680        2688 
0.006080078 0.006378263 0.007252053 0.006726745 0.005745347 0.006470764 0.007158112 0.005672314 
       2699        2772        2777        2779        2798        2801        2812        2840 
0.007016065 0.013464526 0.012846622 0.006384951 0.004748024 0.006435227 0.007487716 0.008115731 
       2849        2852        2856        2900        2910        2912        2946        2948 
0.006556965 0.004882351 0.006384951 0.005156491 0.006091693 0.004787344 0.005054188 0.004801962 
       2965        2969        2971        2990        2994        2995        3004        3008 
0.005054630 0.005674515 0.004891501 0.005016018 0.005494005 0.009165489 0.004821646 0.005616506 
       3012        3018        3021        3030        3041        3046        3051        3052 
0.005184523 0.007585533 0.007383966 0.008916163 0.016455429 0.005893423 0.008984307 0.005310204 
       3065        3074        3126        3130        3155        3188        3221        3274 
0.012183875 0.004946746 0.019492710 0.005384397 0.007093095 0.015836977 0.005940855 0.005995139 
       3292        3309        3324        3332        3347        3367        3378        3381 
0.004646160 0.005018396 0.005633797 0.008055287 0.007555507 0.005219474 0.005844088 0.007215437 
       3413        3418        3419        3446        3469        3474        3487        3496 
0.007952008 0.004995318 0.005468012 0.004726109 0.005565027 0.004674916 0.005068929 0.009369520 
       3519        3524        3526        3545        3549        3557        3587        3611 
0.005543214 0.004635661 0.013422322 0.004757291 0.004918820 0.007684917 0.004872856 0.004872856 
       3622        3650        3666        3709        3710        3720        3721        3727 
0.004733307 0.005857163 0.006405776 0.006141910 0.005615658 0.006389325 0.016139547 0.006618658 
       3741        3752        3763        3769        3787        3792        3807        3818 
0.004870547 0.016330934 0.005287558 0.004728145 0.004943798 0.007689718 0.005998108 0.010775791 
       3820        3823        3824        3835        3843        3848        3854        3863 
0.007366569 0.323058874 0.012258040 0.004967080 0.005005760 0.006333501 0.007689718 0.008161154 
       3890        3895        3912        3917        3922        3953        3991        4024 
0.005616506 0.019354101 0.004912963 0.007690767 0.010025916 0.019421769 0.004912256 0.005151978 
       4035        4071        4088        4112        4125        4129        4131        4132 
0.007969759 0.033246417 0.004918863 0.007559978 0.005498808 0.004805200 0.007131721 0.019492710 
       4140        4143        4174        4180        4191        4195        4260        4287 
0.009613280 0.004686940 0.006136951 0.005524176 0.005514261 0.005146048 0.009759507 0.004801486 
       4300        4301        4343        4362        4401        4463        4482        4493 
0.005225384 0.005310204 0.007165155 0.005507166 0.005566348 0.017712588 0.008767404 0.007154999 
       4511        4512        4547        4553        4558        4585        4627        4633 
0.006995196 0.005054630 0.009347506 0.010326676 0.006875101 0.004851478 0.014885235 0.005546699 
       4639        4687        4693        4696        4710        4715        4754        4759 
0.004777701 0.006998472 0.004640983 0.005081857 0.005440630 0.005196371 0.006386644 0.007418423 
       4790        4829        4873        4902        4908        4972        4992        5040 
0.004856461 0.006130203 0.012769233 0.005893423 0.008892413 0.004912256 0.006275606 0.004889259 
       5041        5054        5087        5107        5154        5178        5187 
0.007383966 0.006838011 0.004875721 0.006541276 0.007677790 0.006034090 0.005565063 

outlier

reduced_4.res <- reduced_4$residuals
crit<-qt(1-0.05/(2*n), n-p-1)
outliers <- reduced_4.res[abs(reduced_4.res)>crit]
outliers
          8          12          21          30          83          85          86         101 
  -5.913799 -112.810937   -5.131140  -28.388375    7.064053   -4.902421    5.268800   -6.738462 
        115         122         142         148         149         154         166         183 
   5.835978  -11.281066   -4.900743   -5.950818   -8.284466   -4.614577    5.536043   18.759111 
        202         206         218         245         264         281         288         372 
 -11.064846   -6.976696    7.906909   -7.166119    6.562536    4.514517   -9.756502   -9.170360 
        373         440         505         509         525         548         566         589 
  -4.811386   -4.829656   10.431685   -5.199469   32.192103  -42.498339   10.705696   -4.616565 
        629         655         657         659         675         687         701         706 
  -5.057653    4.903313  -10.049213   -4.454203   -5.471326   -6.529182   -9.150344   -8.992792 
        717         721         722         725         739         751         768         774 
  -7.819105   -8.206050  -11.839559   -6.756563    4.469453    9.211716   -5.691134   -4.977692 
        788         794         795         817         819         833         840         844 
   5.293240   -5.395176   -9.673765  -13.711257   -4.723057  -10.325351   -4.977692    7.896471 
        875         888         902         903         917         954         964         967 
 -13.765446    4.511825   -4.486556    4.815690   -9.997107   11.645455   -5.057653    7.233862 
        991        1039        1077        1081        1107        1124        1133        1156 
  -4.851860  -29.863426   -6.559467   -5.979445  -11.538933   -9.743607   -8.597773 -358.992053 
       1162        1167        1171        1172        1226        1235        1257        1258 
  -5.913799  -13.370172   -5.991466  -13.904782  -24.686598  -22.516690    9.644487    4.622980 
       1263        1270        1305        1318        1344        1356        1387        1412 
   6.011786    8.044160   -5.145541   -5.100743    5.658642   -6.254700   -4.851860   -9.750690 
       1428        1436        1487        1497        1510        1524        1610        1620 
 -11.498443   -8.316040    7.687631    5.004046   -7.060659    4.482981  -16.343618   -5.238351 
       1652        1658        1665        1669        1693        1706        1776        1787 
 -11.418067   -7.852930   -5.163340  -16.484320   -4.689106   -4.877840  -16.318931   -5.613721 
       1790        1840        1930        1951        1958        1998        2004        2006 
   9.189242   -4.798258   -6.274087   -7.515991   -6.682392    9.251057    4.599440    5.240035 
       2018        2040        2041        2059        2079        2115        2119        2148 
   7.500553    4.734365   -5.264313  -15.336442   -6.043512  -23.388238   -4.859254   -6.605338 
       2156        2167        2195        2205        2220        2256        2258        2262 
  -5.287846   -9.600193   -5.857549   -4.579628    4.948918   -9.053372   17.356606   -7.264594 
       2284        2290        2313        2329        2334        2335        2337        2386 
 -16.484320    5.684278   -4.851860   -7.041619   -7.417245   -4.555771   -5.816303  -12.954904 
       2439        2452        2480        2485        2506        2521        2536        2579 
   9.264406   -8.282663   -4.793832  -18.780311    4.541019  -16.318931   -4.884719    9.211716 
       2594        2628        2629        2642        2660        2661        2666        2696 
  -5.238351   -6.381604    4.489202   -5.096416    4.453651   -5.461202   -6.254700   -6.976696 
       2702        2718        2727        2735        2745        2763        2772        2781 
   7.233862   -6.726264    5.198906   -6.197960   -6.606402  -11.892705    4.431016   -8.362063 
       2793        2795        2802        2840        2866        2892        2898        2955 
  -4.697738   -4.530220   -7.010108    4.976958   -7.349414   11.007314   -4.528817  -14.101843 
       2978        2994        2997        3005        3025        3040        3042        3065 
 -13.426497    6.463923  -14.095802    5.659028   -4.737794   -8.881971   -6.738462   -7.313679 
       3072        3079        3090        3105        3133        3149        3196        3210 
  -6.423514   16.073667    5.085934   -5.548543   -5.466040   -5.426271   -6.355608   -7.747739 
       3243        3248        3257        3264        3322        3336        3343        3351 
  -6.033627    5.580542   -4.595999   -5.857549   -9.308640    9.264406   -4.501913    5.867902 
       3362        3369        3400        3410        3414        3424        3434        3461 
 -13.219174  -12.293034   -4.896657   -5.318501   -9.993826   -6.001601  -10.863890   -6.353654 
       3522        3564        3566        3571        3582        3605        3629        3638 
  -8.866347   -8.604661   -6.119343   -6.889088   -5.322449   -9.260519   -8.348111   -6.028213 
       3711        3716        3740        3750        3784        3816        3841        3853 
  -6.506898    5.779253   -7.586267    5.060560   -4.746179   -6.432015   -6.569981    5.240035 
       3855        3866        3903        3910        3913        3915        3942        3947 
 -10.826648   -4.935304    5.739331   -9.056519   -4.882192  -16.323382   -5.403135   -4.485039 
       3951        4003        4019        4090        4093        4111        4116        4127 
  -6.092031   -8.881971   -4.877840   -4.542401   -5.913799  -15.362967   -5.909070   -7.854482 
       4144        4145        4151        4156        4159        4170        4233        4243 
   4.541019   -8.716350   -9.593344   -7.055209  -42.578664   -4.837952   -4.614577   -5.850980 
       4248        4306        4311        4329        4354        4356        4460        4490 
 -14.114484   -5.627580    4.678171  -11.261020  -14.672621   -5.417642    6.046933   -5.950818 
       4503        4505        4527        4544        4547        4556        4580        4610 
  -6.716718    4.453651    4.477563   -8.992792   -4.960846   -4.813642  -13.243536    5.353589 
       4614        4627        4650        4656        4659        4677        4701        4713 
  -5.833289   -4.965536   -4.896657    4.644296   -6.288973    5.012168  -10.498461    8.507914 
       4729        4783        4794        4815        4858        4880        4887        4910 
  -6.726264    7.580155   -5.470226   -5.334876   -5.096416   -6.738462    5.612130   -6.821823 
       4924        4939        4988        5045        5049        5051        5091        5095 
  -4.543092    4.488823   -6.821823  -11.196377   -4.972743   -6.197392   -8.362063   -7.809710 
       5133        5140        5179        5195 
  -5.399943    4.815690   -8.541941  -10.162346 
## outliers removed
outliers_index <- attr(outliers, "names")
outliers_index <- as.numeric(outliers_index)
train_no_outliers <- train[-(outliers_index),]

#leverages removed
lererages_index <- attr(leverages, "names")
lererages_index <- as.numeric(lererages_index)
train_no_leverages <- train[-(lererages_index),]

# DDFFITS_influence
DDFFITS_index <- attr(DDFFITS_influence, "names")
DDFFITS_index <- as.numeric(DDFFITS_index)
train_no_DDFFITS <- train[-(DDFFITS_index),]

# all "non-normal" removed
all_special <- c(DDFFITS_index,lererages_index,outliers_index)
train_nothing_special <- train[-(all_special),]
train_nothing_special
NA
NA
NA
NA
vif(train[c(2,3,4,7,8,6,9,10,11)])
    volatile.acidity          citric.acid       residual.sugar total.sulfur.dioxide 
            1.818602             1.505238             3.353752             2.850601 
             density  free.sulfur.dioxide                   pH            sulphates 
            5.664883             2.099196             1.338240             1.427424 
             alcohol 
            2.890129 
train_temp<-train
# as.factor(train_temp$Type)<-numeric(train_temp$Type)
#train_temp
# as.factor

train_temp$Type <- as.numeric(train_temp$Type)-1
train_temp$Type <- as.integer(train_temp$Type)
train_temp

creating reduced

reduced_4_3 <- glm(cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type, family=binomial, data=train_no_outliers)

reduced_4_4_lev <- glm(cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type, family=binomial, data=train_no_leverages)

reduced_4_5_DDFFITS <- glm(cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type, family=binomial, data=train_no_DDFFITS)

reduced_4_6_no_special <- glm(cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type, family=binomial, data=train_nothing_special)
# summary(reduced_4_6_no_special)
## checking colinearity / VIF scores
# reduced_4_3_col <- data.frame('reduced_4_3' = check_collinearity(reduced_4_3))
# reduced_4_3_col_VIF <- reduced_4_3_col[c('reduced_4_3.Term','reduced_4_3.VIF')]
# reduced_4_3_col_VIF
# 
# reduced_4_4_lev_col <- data.frame('reduced_4_4_lev' = check_collinearity(reduced_4_4_lev))
# reduced_4_4_lev_col_VIF <- reduced_4_4_lev_col[c('reduced_4_4_lev.Term','reduced_4_4_lev.VIF')]
# reduced_4_4_lev_col_VIF
# 
# 
# reduced_4_5_DDFFITS_col <- data.frame('reduced_4_5_DDFFITS' = check_collinearity(reduced_4_5_DDFFITS))
# reduced_4_5_DDFFITS_col_VIF <- reduced_4_5_DDFFITS_col[c('reduced_4_5_DDFFITS.Term','reduced_4_5_DDFFITS.VIF')]
# reduced_4_5_DDFFITS_col_VIF
# 
# reduced_4_6_no_special_col <- data.frame('reduced_4_6_no_special' = check_collinearity(reduced_4_6_no_special))
# reduced_4_6_no_special_col_VIF <- reduced_4_6_no_special_col[c('reduced_4_6_no_special.Term','reduced_4_6_no_special.VIF')]
# reduced_4_6_no_special_col_VIF
# 
# VIF_summary <- data.frame('0'=reduced_4_3_col_VIF['reduced_4_3.Term'],
#                           '1'=reduced_4_3_col_VIF['reduced_4_3.VIF'],
#                           '2'=reduced_4_4_lev_col_VIF['reduced_4_4_lev.VIF'],
#                           '3'=reduced_4_5_DDFFITS_col_VIF['reduced_4_5_DDFFITS.VIF'],
#                           '4'=reduced_4_6_no_special_col_VIF['reduced_4_6_no_special.VIF'])
# colnames(VIF_summary) <- c('Predictor Variable','4_3.VIF.Outliers','4_4_lev.VIF','4_5_DDFFITS.VIF','4_6_no_special.VIF')
# VIF_summary

## VIF for Outliers
### cat_quality~volatile.acidity+citric.acid+residual.sugar+total.sulfur.dioxide+density+free.sulfur.dioxide+pH+sulphates+alcohol+Type


#
reg_4_VIF_test <- vif(train_temp[c(1,2,3,4,7,8,6,9,10,11,12)])
reg_4_2_VIF_test <- vif(train_temp[c(2,3,4,7,8,6,9,10,11,12)])
outliers_VIF <- vif(train_no_outliers[c(2,3,4,7,8,6,9,10,11,12)])
leverage_VIF <- vif(train_no_leverages[c(2,3,4,7,8,6,9,10,11,12)])
DDFFITS_VIF <- vif(train_no_DDFFITS[c(2,3,4,7,8,6,9,10,11,12)])
nothing_special <- vif(train_nothing_special[c(2,3,4,7,8,6,9,10,11,12)])


reg_4_VIF_test
       fixed.acidity     volatile.acidity          citric.acid       residual.sugar 
            4.962487             2.139008             1.585795             9.280982 
total.sulfur.dioxide              density  free.sulfur.dioxide                   pH 
            3.974170            21.338984             2.189173             2.483275 
           sulphates              alcohol                 Type 
            1.514014             5.371286             7.045601 
VIF_summary_test <- data.frame('best_possible_VIF (post)'=reg_4_2_VIF_test,
                               'outliers_VIF'=outliers_VIF,
                               'leverage_VIF'=leverage_VIF,
                               'DDFFITS_VIF'= DDFFITS_VIF,
                               'nothing_special'=nothing_special)
VIF_summary_test
NA
##evaluating model
Reduced4_3_AIC_train <- reduced_4_3$aic

##predicted quality for test data based on training data
preds<-predict(reduced_4_3,newdata=test, type="response")

reduced_4_3_error <- table(test$cat_quality, preds>0.6)

evulation_summary_4_3 <- data.frame(
  attempt = 'reduced_4_3_error_outliers',
  AIC = Reduced4_3_AIC_train,
  PRESS = get_press(reduced_4_3),
  'False positive' = round(reduced_4_3_error[3]/(reduced_4_3_error[1]+reduced_4_3_error[3]),3),
  'False negative' = round(reduced_4_3_error[2]/(reduced_4_3_error[2]+reduced_4_3_error[4]),3),
  'Error Rate' = round((reduced_4_3_error[2]+reduced_4_3_error[3])/(reduced_4_3_error[1]+reduced_4_3_error[2]+reduced_4_3_error[3]+reduced_4_3_error[4]),3)
)

evulation_summary <- rbind(evulation_summary,evulation_summary_4_3)
evulation_summary
##evaluating model leverage
reduced_4_4_lev_AIC_train <- reduced_4_4_lev$aic

##predicted quality for test data based on training data
preds<-predict(reduced_4_4_lev,newdata=test, type="response")

reduced_4_4_lev_error <- table(test$cat_quality, preds>0.65)

evulation_summary_4_4_lev <- data.frame(
  attempt = 'reduced_4_4_lev_error',
  AIC = reduced_4_4_lev_AIC_train,
  PRESS = get_press(reduced_4_4_lev),
  'False positive' = round(reduced_4_4_lev_error[3]/(reduced_4_4_lev_error[1]+reduced_4_4_lev_error[3]),3),
  'False negative' = round(reduced_4_4_lev_error[2]/(reduced_4_4_lev_error[2]+reduced_4_4_lev_error[4]),3),
  'Error Rate' = round((reduced_4_4_lev_error[2]+reduced_4_4_lev_error[3])/(reduced_4_4_lev_error[1]+reduced_4_4_lev_error[2]+reduced_4_4_lev_error[3]+reduced_4_4_lev_error[4]),3)
)

evulation_summary <- rbind(evulation_summary,evulation_summary_4_4_lev)
evulation_summary
##evaluating model DDFFITS
reduced_4_5_DDFFITS_AIC_train <- reduced_4_5_DDFFITS$aic

##predicted quality for test data based on training data
preds<-predict(reduced_4_5_DDFFITS,newdata=test, type="response")

reduced_4_5_DDFFITS_error <- table(test$cat_quality, preds>0.7)

evulation_summary_4_5_DDFFITS <- data.frame(
  attempt = 'reduced_4_5_DDFFITS_error',
  AIC = reduced_4_5_DDFFITS_AIC_train,
  PRESS = get_press(reduced_4_5_DDFFITS),
  'False positive' = round(reduced_4_5_DDFFITS_error[3]/(reduced_4_5_DDFFITS_error[1]+reduced_4_5_DDFFITS_error[3]),3),
  'False negative' = round(reduced_4_5_DDFFITS_error[2]/(reduced_4_5_DDFFITS_error[2]+reduced_4_5_DDFFITS_error[4]),3),
  'Error Rate' = round((reduced_4_5_DDFFITS_error[2]+reduced_4_5_DDFFITS_error[3])/(reduced_4_5_DDFFITS_error[1]+reduced_4_5_DDFFITS_error[2]+reduced_4_5_DDFFITS_error[3]+reduced_4_5_DDFFITS_error[4]),3)
)

evulation_summary <- rbind(evulation_summary,evulation_summary_4_5_DDFFITS)
evulation_summary
##evaluating model DDFFITS
reduced_4_6_no_special_AIC_train <- reduced_4_6_no_special$aic

##predicted quality for test data based on training data
preds<-predict(reduced_4_6_no_special,newdata=test, type="response")

reduced_4_6_no_special_error <- table(test$cat_quality, preds>0.8)

evulation_summary_4_6_no_special <- data.frame(
  attempt = 'reduced_4_6_no_special_error',
  AIC = reduced_4_6_no_special_AIC_train,
  PRESS = get_press(reduced_4_6_no_special),
  'False positive' = round(reduced_4_6_no_special_error[3]/(reduced_4_6_no_special_error[1]+reduced_4_6_no_special_error[3]),3),
  'False negative' = round(reduced_4_6_no_special_error[2]/(reduced_4_6_no_special_error[2]+reduced_4_6_no_special_error[4]),3),
  'Error Rate' = round((reduced_4_6_no_special_error[2]+reduced_4_6_no_special_error[3])/(reduced_4_6_no_special_error[1]+reduced_4_6_no_special_error[2]+reduced_4_6_no_special_error[3]+reduced_4_6_no_special_error[4]),3)
)

evulation_summary <- rbind(evulation_summary,evulation_summary_4_6_no_special)
evulation_summary

ROC Curves and AUC

## reduced_1
# detach(package:performance, unload=TRUE)
## FYI the performance package causes ROC curves to not work
library(ROCR)



# reduced_1
preds<-predict(reduced_1,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_1")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
reduced_1_auc <- auc@y.values

## reduced_4
preds<-predict(reduced_4,newdata=test, type="response")
rates4<-prediction(preds, test$cat_quality)
roc_result<-performance(rates4,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4")
lines(x = c(0,1), y = c(0,1), col="red")


auc4<-performance(rates4, measure = "auc")
reduced_4_auc <- auc4@y.values

## reduced_4_2
preds<-predict(reduced_4_2,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_2")
lines(x = c(0,1), y = c(0,1), col="red")


auc4_2<-performance(rates, measure = "auc")
reduced_4_2_auc <- auc4_2@y.values

## reduced_4_3
preds<-predict(reduced_4_3,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_3")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
reduced_4_3_auc <- auc@y.values

## reduced_4_4_lev 
preds<-predict(reduced_4_4_lev,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_4_lev")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
reduced_4_4_lev_auc <- auc@y.values

## reduced_4_5_DDFFITS 
preds<-predict(reduced_4_5_DDFFITS,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_5_DDFFITS")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
reduced_4_5_DDFFITS_auc <- auc@y.values

## reduced_4_6_no_special 
preds<-predict(reduced_4_6_no_special,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_6_no_special")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
reduced_4_6_no_special_auc <- auc@y.values

AUC_summary <- data.frame('reduced_1'=reduced_1_auc,
                          'reduced_4'=reduced_4_auc,
                          'reduced_4_2'=reduced_4_2_auc,
                          'reduced_4_3'=reduced_4_3_auc,
                          'reduced_4_4_lev'=reduced_4_4_lev_auc,
                          'reduced_4_5_DDFFITS'=reduced_4_5_DDFFITS_auc,
                          'reduced_4_6_no_special'=reduced_4_6_no_special_auc)
colnames(AUC_summary) <- c('reduced_1','reduced_4','reduced_4_2','reduced_4_3','reduced_4_4_lev','reduced_4_5_DDFFITS','reduced_4_6_no_special')

AUC_summary

Ryan’s part starts here

The goal is to make 3 models: One for just white, one for just red, and one with interaction terms with the type of wine.

After that, the models will be trained on the filtered datasets and the resulting scores will be added to the evaluation summary.

Red wine only model

regfull_Red<-glm(cat_quality~., family="binomial", data=train_Red_NoType)
regnull_Red<-glm(cat_quality~1, family="binomial", data=train_Red_NoType)
step(regnull_Red, scope=list(lower=regnull_Red, upper=regfull_Red), direction="forward")
Start:  AIC=1811.89
cat_quality ~ 1

                       Df Deviance    AIC
+ alcohol               1   1540.6 1544.6
+ volatile.acidity      1   1680.6 1684.6
+ total.sulfur.dioxide  1   1726.9 1730.9
+ sulphates             1   1741.1 1745.1
+ citric.acid           1   1778.7 1782.7
+ density               1   1780.0 1784.0
+ chlorides             1   1795.4 1799.4
+ fixed.acidity         1   1797.8 1801.8
+ free.sulfur.dioxide   1   1803.8 1807.8
<none>                      1809.9 1811.9
+ pH                    1   1809.8 1813.8
+ residual.sugar        1   1809.9 1813.9

Step:  AIC=1544.61
cat_quality ~ alcohol

                       Df Deviance    AIC
+ volatile.acidity      1   1450.7 1456.7
+ sulphates             1   1492.7 1498.7
+ total.sulfur.dioxide  1   1507.4 1513.4
+ fixed.acidity         1   1521.6 1527.6
+ citric.acid           1   1523.5 1529.5
+ pH                    1   1528.7 1534.7
+ density               1   1535.8 1541.8
<none>                      1540.6 1544.6
+ free.sulfur.dioxide   1   1539.2 1545.2
+ residual.sugar        1   1539.8 1545.8
+ chlorides             1   1540.5 1546.5

Step:  AIC=1456.66
cat_quality ~ alcohol + volatile.acidity

                       Df Deviance    AIC
+ total.sulfur.dioxide  1   1417.1 1425.1
+ sulphates             1   1427.5 1435.5
+ fixed.acidity         1   1448.1 1456.1
+ free.sulfur.dioxide   1   1448.1 1456.1
<none>                      1450.7 1456.7
+ citric.acid           1   1449.0 1457.0
+ density               1   1449.5 1457.5
+ residual.sugar        1   1449.8 1457.8
+ pH                    1   1449.9 1457.9
+ chlorides             1   1450.7 1458.7

Step:  AIC=1425.14
cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide

                      Df Deviance    AIC
+ sulphates            1   1387.4 1397.4
+ free.sulfur.dioxide  1   1409.8 1419.8
<none>                     1417.1 1425.1
+ pH                   1   1415.8 1425.8
+ density              1   1416.1 1426.1
+ fixed.acidity        1   1416.2 1426.2
+ citric.acid          1   1416.8 1426.8
+ residual.sugar       1   1417.0 1427.0
+ chlorides            1   1417.1 1427.1

Step:  AIC=1397.37
cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide + 
    sulphates

                      Df Deviance    AIC
+ free.sulfur.dioxide  1   1379.6 1391.6
+ chlorides            1   1380.0 1392.0
+ citric.acid          1   1383.9 1395.9
<none>                     1387.4 1397.4
+ residual.sugar       1   1387.1 1399.1
+ fixed.acidity        1   1387.3 1399.3
+ pH                   1   1387.4 1399.4
+ density              1   1387.4 1399.4

Step:  AIC=1391.64
cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide + 
    sulphates + free.sulfur.dioxide

                 Df Deviance    AIC
+ chlorides       1   1372.6 1386.6
<none>                1379.6 1391.6
+ citric.acid     1   1377.9 1391.9
+ pH              1   1379.3 1393.3
+ fixed.acidity   1   1379.3 1393.3
+ residual.sugar  1   1379.5 1393.5
+ density         1   1379.6 1393.6

Step:  AIC=1386.63
cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide + 
    sulphates + free.sulfur.dioxide + chlorides

                 Df Deviance    AIC
<none>                1372.6 1386.6
+ pH              1   1371.2 1387.2
+ citric.acid     1   1372.1 1388.1
+ fixed.acidity   1   1372.2 1388.2
+ residual.sugar  1   1372.2 1388.2
+ density         1   1372.6 1388.6

Call:  glm(formula = cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide + 
    sulphates + free.sulfur.dioxide + chlorides, family = "binomial", 
    data = train_Red_NoType)

Coefficients:
         (Intercept)               alcohol      volatile.acidity  total.sulfur.dioxide  
            -8.02155               0.85512              -2.93115              -0.01843  
           sulphates   free.sulfur.dioxide             chlorides  
             2.63739               0.02326              -4.10141  

Degrees of Freedom: 1308 Total (i.e. Null);  1302 Residual
Null Deviance:      1810 
Residual Deviance: 1373     AIC: 1387

The model looks great after the foward selection! Time to test and add to the evaluation summary.

model1_Red<-glm(formula = cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide + 
    sulphates + free.sulfur.dioxide + chlorides, family = "binomial", 
    data = train_Red_NoType)

summary(model1_Red)

Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide + 
    sulphates + free.sulfur.dioxide + chlorides, family = "binomial", 
    data = train_Red_NoType)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.0655  -0.8628   0.3239   0.8474   2.3082  

Coefficients:
                      Estimate Std. Error z value Pr(>|z|)    
(Intercept)          -8.021553   0.886912  -9.044  < 2e-16 ***
alcohol               0.855123   0.078409  10.906  < 2e-16 ***
volatile.acidity     -2.931148   0.413395  -7.090 1.34e-12 ***
total.sulfur.dioxide -0.018428   0.002926  -6.297 3.03e-10 ***
sulphates             2.637391   0.456973   5.771 7.86e-09 ***
free.sulfur.dioxide   0.023257   0.008608   2.702  0.00689 ** 
chlorides            -4.101411   1.588464  -2.582  0.00982 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 1809.9  on 1308  degrees of freedom
Residual deviance: 1372.6  on 1302  degrees of freedom
AIC: 1386.6

Number of Fisher Scoring iterations: 4

##evaluating model
model1_Red_AIC_train <- model1_Red$aic
##predicted quality for test data based on training data
test_Red_NoType<-subset(test, Type == 0, select=-c(Type))
preds<-predict(model1_Red,newdata=test_Red_NoType, type="response")
model1_Red_error <- table(test_Red_NoType$cat_quality, preds>0.7)
#Curves
evulation_summary_1R <- data.frame(
  attempt = 'model1_Red',
  AIC = model1_Red_AIC_train,
  PRESS = get_press(model1_Red),
  'False positive' = round(model1_Red_error[3]/(model1_Red_error[1]+model1_Red_error[3]),3),
  'False negative' = round(model1_Red_error[2]/(model1_Red_error[2]+model1_Red_error[4]),3),
  'Error Rate' = round((model1_Red_error[2]+model1_Red_error[3])/(model1_Red_error[1]+model1_Red_error[2]+model1_Red_error[3]+model1_Red_error[4]),3)
)

compare_models<-rbind(evulation_summary[1,],evulation_summary_1R)
compare_models

evulation_summary <- rbind(evulation_summary,evulation_summary_1R)
evulation_summary
NA
# model1_Red
library(ROCR)
preds<-predict(model1_Red,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_Red")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
model1_Red_auc <- auc@y.values

White wine only model

regfull_White<-glm(cat_quality~., family="binomial", data=train_White_NoType)
regnull_White<-glm(cat_quality~1,family="binomial", data=train_White_NoType)
step(regnull_White, scope=list(lower=regnull_White, upper=regfull_White), direction="forward")
Start:  AIC=4947.36
cat_quality ~ 1

                       Df Deviance    AIC
+ alcohol               1   4280.3 4284.3
+ density               1   4668.2 4672.2
+ volatile.acidity      1   4758.7 4762.7
+ chlorides             1   4805.4 4809.4
+ total.sulfur.dioxide  1   4831.9 4835.9
+ fixed.acidity         1   4912.0 4916.0
+ pH                    1   4914.3 4918.3
+ residual.sugar        1   4919.1 4923.1
+ sulphates             1   4934.7 4938.7
<none>                      4945.4 4947.4
+ free.sulfur.dioxide   1   4945.2 4949.2
+ citric.acid           1   4945.4 4949.4

Step:  AIC=4284.29
cat_quality ~ alcohol

                       Df Deviance    AIC
+ volatile.acidity      1   4030.1 4036.1
+ residual.sugar        1   4214.8 4220.8
+ free.sulfur.dioxide   1   4235.1 4241.1
+ density               1   4253.7 4259.7
+ sulphates             1   4261.3 4267.3
+ fixed.acidity         1   4267.3 4273.3
+ chlorides             1   4273.1 4279.1
+ pH                    1   4276.2 4282.2
+ citric.acid           1   4276.3 4282.3
<none>                      4280.3 4284.3
+ total.sulfur.dioxide  1   4279.4 4285.4

Step:  AIC=4036.1
cat_quality ~ alcohol + volatile.acidity

                       Df Deviance    AIC
+ residual.sugar        1   3942.1 3950.1
+ density               1   3984.6 3992.6
+ free.sulfur.dioxide   1   3999.3 4007.3
+ sulphates             1   4013.2 4021.2
+ fixed.acidity         1   4014.8 4022.8
+ total.sulfur.dioxide  1   4020.2 4028.2
<none>                      4030.1 4036.1
+ pH                    1   4028.3 4036.3
+ chlorides             1   4028.6 4036.6
+ citric.acid           1   4030.1 4038.1

Step:  AIC=3950.11
cat_quality ~ alcohol + volatile.acidity + residual.sugar

                       Df Deviance    AIC
+ fixed.acidity         1   3923.0 3933.0
+ sulphates             1   3924.2 3934.2
+ free.sulfur.dioxide   1   3930.2 3940.2
+ density               1   3932.9 3942.9
+ pH                    1   3934.3 3944.3
<none>                      3942.1 3950.1
+ total.sulfur.dioxide  1   3941.1 3951.1
+ citric.acid           1   3941.6 3951.6
+ chlorides             1   3942.1 3952.1

Step:  AIC=3932.95
cat_quality ~ alcohol + volatile.acidity + residual.sugar + fixed.acidity

                       Df Deviance    AIC
+ sulphates             1   3904.7 3916.7
+ free.sulfur.dioxide   1   3914.0 3926.0
<none>                      3923.0 3933.0
+ pH                    1   3921.5 3933.5
+ total.sulfur.dioxide  1   3921.7 3933.7
+ density               1   3921.9 3933.9
+ citric.acid           1   3922.7 3934.7
+ chlorides             1   3923.0 3935.0

Step:  AIC=3916.74
cat_quality ~ alcohol + volatile.acidity + residual.sugar + fixed.acidity + 
    sulphates

                       Df Deviance    AIC
+ free.sulfur.dioxide   1   3896.9 3910.9
+ density               1   3898.2 3912.2
<none>                      3904.7 3916.7
+ pH                    1   3904.6 3918.6
+ total.sulfur.dioxide  1   3904.6 3918.6
+ citric.acid           1   3904.7 3918.7
+ chlorides             1   3904.7 3918.7

Step:  AIC=3910.91
cat_quality ~ alcohol + volatile.acidity + residual.sugar + fixed.acidity + 
    sulphates + free.sulfur.dioxide

                       Df Deviance    AIC
+ density               1   3890.7 3906.7
+ total.sulfur.dioxide  1   3894.7 3910.7
<none>                      3896.9 3910.9
+ pH                    1   3896.8 3912.8
+ chlorides             1   3896.9 3912.9
+ citric.acid           1   3896.9 3912.9

Step:  AIC=3906.7
cat_quality ~ alcohol + volatile.acidity + residual.sugar + fixed.acidity + 
    sulphates + free.sulfur.dioxide + density

                       Df Deviance    AIC
+ pH                    1   3885.2 3903.2
<none>                      3890.7 3906.7
+ total.sulfur.dioxide  1   3889.6 3907.6
+ chlorides             1   3890.7 3908.7
+ citric.acid           1   3890.7 3908.7

Step:  AIC=3903.24
cat_quality ~ alcohol + volatile.acidity + residual.sugar + fixed.acidity + 
    sulphates + free.sulfur.dioxide + density + pH

                       Df Deviance    AIC
<none>                      3885.2 3903.2
+ total.sulfur.dioxide  1   3884.3 3904.3
+ chlorides             1   3885.0 3905.0
+ citric.acid           1   3885.1 3905.1

Call:  glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar + 
    fixed.acidity + sulphates + free.sulfur.dioxide + density + 
    pH, family = "binomial", data = train_White_NoType)

Coefficients:
        (Intercept)              alcohol     volatile.acidity       residual.sugar  
          2.041e+02            8.487e-01           -6.416e+00            1.563e-01  
      fixed.acidity            sulphates  free.sulfur.dioxide              density  
         -9.407e-03            1.877e+00            6.778e-03           -2.165e+02  
                 pH  
          8.920e-01  

Degrees of Freedom: 3887 Total (i.e. Null);  3879 Residual
Null Deviance:      4945 
Residual Deviance: 3885     AIC: 3903

The model looks good after the foward selection, but the predictor fixed.acidity can be removed. The density VIF is above ten, but jsut barely. For now, it will be left in. Time to test and add to the evaluation summary.

model1_White<-glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar + 
    fixed.acidity + sulphates + free.sulfur.dioxide + density + 
    pH, family = "binomial", data = train_White_NoType)


summary(model1_White)

Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar + 
    fixed.acidity + sulphates + free.sulfur.dioxide + density + 
    pH, family = "binomial", data = train_White_NoType)

Deviance Residuals: 
   Min      1Q  Median      3Q     Max  
-3.198  -0.888   0.437   0.798   2.507  

Coefficients:
                      Estimate Std. Error z value Pr(>|z|)    
(Intercept)          2.041e+02  6.993e+01   2.918  0.00352 ** 
alcohol              8.487e-01  9.686e-02   8.763  < 2e-16 ***
volatile.acidity    -6.416e+00  4.477e-01 -14.329  < 2e-16 ***
residual.sugar       1.563e-01  2.746e-02   5.689 1.27e-08 ***
fixed.acidity       -9.407e-03  7.582e-02  -0.124  0.90126    
sulphates            1.877e+00  4.028e-01   4.661 3.15e-06 ***
free.sulfur.dioxide  6.778e-03  2.540e-03   2.669  0.00761 ** 
density             -2.165e+02  7.088e+01  -3.054  0.00226 ** 
pH                   8.920e-01  3.886e-01   2.295  0.02172 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 4945.4  on 3887  degrees of freedom
Residual deviance: 3885.2  on 3879  degrees of freedom
AIC: 3903.2

Number of Fisher Scoring iterations: 5
model1_White<-glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar + sulphates + free.sulfur.dioxide + density + 
    pH, family = "binomial", data = train_White_NoType)

summary(model1_White)

Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar + 
    sulphates + free.sulfur.dioxide + density + pH, family = "binomial", 
    data = train_White_NoType)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.1956  -0.8867   0.4377   0.7961   2.5070  

Coefficients:
                      Estimate Std. Error z value Pr(>|z|)    
(Intercept)          2.105e+02  4.762e+01   4.420 9.87e-06 ***
alcohol              8.408e-01  7.311e-02  11.501  < 2e-16 ***
volatile.acidity    -6.407e+00  4.417e-01 -14.504  < 2e-16 ***
residual.sugar       1.586e-01  1.972e-02   8.045 8.66e-16 ***
sulphates            1.885e+00  3.978e-01   4.739 2.14e-06 ***
free.sulfur.dioxide  6.796e-03  2.536e-03   2.680  0.00737 ** 
density             -2.230e+02  4.773e+01  -4.673 2.97e-06 ***
pH                   9.247e-01  2.859e-01   3.235  0.00122 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 4945.4  on 3887  degrees of freedom
Residual deviance: 3885.3  on 3880  degrees of freedom
AIC: 3901.3

Number of Fisher Scoring iterations: 5

##evaluating model
model1_White_AIC_train <- model1_White$aic
##predicted quality for test data based on training data
test_White_NoType<-subset(test, Type == 1, select=-c(Type))
preds<-predict(model1_White,newdata=test_White_NoType, type="response")
model1_White_error <- table(test_White_NoType$cat_quality, preds>0.7)
#Curves
evulation_summary_1W <- data.frame(
  attempt = 'model1_White',
  AIC = model1_White_AIC_train,
  PRESS = get_press(model1_White),
  'False positive' = round(model1_White_error[3]/(model1_White_error[1]+model1_White_error[3]),3),
  'False negative' = round(model1_White_error[2]/(model1_White_error[2]+model1_White_error[4]),3),
  'Error Rate' = round((model1_White_error[2]+model1_White_error[3])/(model1_White_error[1]+model1_White_error[2]+model1_White_error[3]+model1_White_error[4]),3)
)

evulation_summary <- rbind(evulation_summary,evulation_summary_1W)
evulation_summary


compare_models<-rbind(compare_models,evulation_summary_1W)
compare_models
NA

The model with interaction terms

regfull_int<-glm(cat_quality~.*Type, family="binomial", data=train)
regnull_int<-glm(cat_quality~1,family="binomial", data=train)
step(regnull_int, scope=list(lower=regnull_int, upper=regfull_int), direction="forward")
Start:  AIC=6835.15
cat_quality ~ 1

                       Df Deviance    AIC
+ alcohol               1   5904.3 5908.3
+ density               1   6450.6 6454.6
+ volatile.acidity      1   6468.6 6472.6
+ chlorides             1   6640.0 6644.0
+ Type                  1   6755.3 6759.3
+ citric.acid           1   6801.3 6805.3
+ fixed.acidity         1   6807.6 6811.6
+ free.sulfur.dioxide   1   6822.7 6826.7
+ total.sulfur.dioxide  1   6823.5 6827.5
+ sulphates             1   6827.8 6831.8
+ residual.sugar        1   6830.8 6834.8
<none>                      6833.2 6835.2
+ pH                    1   6831.4 6835.4

Step:  AIC=5908.3
cat_quality ~ alcohol

                       Df Deviance    AIC
+ volatile.acidity      1   5539.7 5545.7
+ residual.sugar        1   5781.4 5787.4
+ free.sulfur.dioxide   1   5810.8 5816.8
+ Type                  1   5826.6 5832.6
+ chlorides             1   5861.3 5867.3
+ citric.acid           1   5864.8 5870.8
+ total.sulfur.dioxide  1   5872.3 5878.3
+ fixed.acidity         1   5893.0 5899.0
+ pH                    1   5895.5 5901.5
+ sulphates             1   5897.6 5903.6
<none>                      5904.3 5908.3
+ density               1   5903.5 5909.5

Step:  AIC=5545.67
cat_quality ~ alcohol + volatile.acidity

                       Df Deviance    AIC
+ density               1   5476.5 5484.5
+ sulphates             1   5477.1 5485.1
+ residual.sugar        1   5500.5 5508.5
+ Type                  1   5504.4 5512.4
+ total.sulfur.dioxide  1   5524.4 5532.4
+ pH                    1   5532.9 5540.9
+ free.sulfur.dioxide   1   5533.7 5541.7
<none>                      5539.7 5545.7
+ chlorides             1   5537.8 5545.8
+ citric.acid           1   5537.9 5545.9
+ fixed.acidity         1   5538.6 5546.6

Step:  AIC=5484.51
cat_quality ~ alcohol + volatile.acidity + density

                       Df Deviance    AIC
+ sulphates             1   5442.1 5452.1
+ citric.acid           1   5463.1 5473.1
+ fixed.acidity         1   5463.8 5473.8
+ total.sulfur.dioxide  1   5464.0 5474.0
+ Type                  1   5465.2 5475.2
+ pH                    1   5471.0 5481.0
+ free.sulfur.dioxide   1   5471.2 5481.2
+ residual.sugar        1   5472.8 5482.8
<none>                      5476.5 5484.5
+ chlorides             1   5476.3 5486.3

Step:  AIC=5452.14
cat_quality ~ alcohol + volatile.acidity + density + sulphates

                       Df Deviance    AIC
+ residual.sugar        1   5419.3 5431.3
+ fixed.acidity         1   5422.1 5434.1
+ citric.acid           1   5424.1 5436.1
+ free.sulfur.dioxide   1   5432.7 5444.7
+ total.sulfur.dioxide  1   5435.6 5447.6
+ pH                    1   5439.3 5451.3
+ chlorides             1   5439.4 5451.4
<none>                      5442.1 5452.1
+ Type                  1   5440.5 5452.5

Step:  AIC=5431.3
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar

                       Df Deviance    AIC
+ Type                  1   5379.2 5393.2
+ total.sulfur.dioxide  1   5385.9 5399.9
+ citric.acid           1   5404.8 5418.8
+ pH                    1   5410.7 5424.7
+ fixed.acidity         1   5415.4 5429.4
<none>                      5419.3 5431.3
+ free.sulfur.dioxide   1   5417.5 5431.5
+ chlorides             1   5418.8 5432.8

Step:  AIC=5393.24
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type

                        Df Deviance    AIC
+ volatile.acidity:Type  1   5348.7 5364.7
+ total.sulfur.dioxide   1   5371.4 5387.4
+ pH                     1   5371.9 5387.9
+ citric.acid            1   5372.7 5388.7
+ free.sulfur.dioxide    1   5373.9 5389.9
+ residual.sugar:Type    1   5374.6 5390.6
+ chlorides              1   5376.4 5392.4
+ density:Type           1   5377.1 5393.1
<none>                       5379.2 5393.2
+ fixed.acidity          1   5378.4 5394.4
+ alcohol:Type           1   5379.2 5395.2
+ sulphates:Type         1   5379.2 5395.2

Step:  AIC=5364.67
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + volatile.acidity:Type

                       Df Deviance    AIC
+ total.sulfur.dioxide  1   5342.0 5360.0
+ residual.sugar:Type   1   5343.3 5361.3
+ free.sulfur.dioxide   1   5344.5 5362.5
+ pH                    1   5344.7 5362.7
+ density:Type          1   5345.2 5363.2
+ chlorides             1   5345.8 5363.8
+ citric.acid           1   5346.2 5364.2
<none>                      5348.7 5364.7
+ alcohol:Type          1   5348.3 5366.3
+ sulphates:Type        1   5348.5 5366.5
+ fixed.acidity         1   5348.6 5366.6

Step:  AIC=5359.98
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + total.sulfur.dioxide + volatile.acidity:Type

                            Df Deviance    AIC
+ total.sulfur.dioxide:Type  1   5303.1 5323.1
+ free.sulfur.dioxide        1   5323.1 5343.1
+ residual.sugar:Type        1   5337.0 5357.0
+ pH                         1   5337.3 5357.3
+ chlorides                  1   5338.9 5358.9
+ density:Type               1   5339.4 5359.4
+ citric.acid                1   5339.9 5359.9
<none>                           5342.0 5360.0
+ alcohol:Type               1   5341.7 5361.7
+ fixed.acidity              1   5341.8 5361.8
+ sulphates:Type             1   5341.9 5361.9

Step:  AIC=5323.07
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + total.sulfur.dioxide + volatile.acidity:Type + 
    Type:total.sulfur.dioxide

                      Df Deviance    AIC
+ free.sulfur.dioxide  1   5286.9 5308.9
+ density:Type         1   5295.6 5317.6
+ pH                   1   5299.6 5321.6
+ alcohol:Type         1   5300.2 5322.2
+ chlorides            1   5300.3 5322.3
<none>                     5303.1 5323.1
+ citric.acid          1   5302.1 5324.1
+ sulphates:Type       1   5302.2 5324.2
+ residual.sugar:Type  1   5302.4 5324.4
+ fixed.acidity        1   5302.9 5324.9

Step:  AIC=5308.91
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide + 
    volatile.acidity:Type + Type:total.sulfur.dioxide

                           Df Deviance    AIC
+ density:Type              1   5281.0 5305.0
+ chlorides                 1   5283.6 5307.6
+ alcohol:Type              1   5283.8 5307.8
+ pH                        1   5284.1 5308.1
<none>                          5286.9 5308.9
+ free.sulfur.dioxide:Type  1   5285.6 5309.6
+ citric.acid               1   5285.8 5309.8
+ residual.sugar:Type       1   5286.2 5310.2
+ sulphates:Type            1   5286.3 5310.3
+ fixed.acidity             1   5286.8 5310.8

Step:  AIC=5305.04
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide + 
    volatile.acidity:Type + Type:total.sulfur.dioxide + density:Type

                           Df Deviance    AIC
+ pH                        1   5275.6 5301.6
+ residual.sugar:Type       1   5276.7 5302.7
+ chlorides                 1   5277.1 5303.1
<none>                          5281.0 5305.0
+ citric.acid               1   5279.1 5305.1
+ free.sulfur.dioxide:Type  1   5279.2 5305.2
+ fixed.acidity             1   5280.0 5306.0
+ alcohol:Type              1   5280.6 5306.6
+ sulphates:Type            1   5280.9 5306.9

Step:  AIC=5301.61
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide + 
    pH + volatile.acidity:Type + Type:total.sulfur.dioxide + 
    density:Type

                           Df Deviance    AIC
+ residual.sugar:Type       1   5270.3 5298.3
+ pH:Type                   1   5271.9 5299.9
+ chlorides                 1   5272.9 5300.9
<none>                          5275.6 5301.6
+ free.sulfur.dioxide:Type  1   5274.3 5302.3
+ fixed.acidity             1   5275.0 5303.0
+ citric.acid               1   5275.1 5303.1
+ sulphates:Type            1   5275.2 5303.2
+ alcohol:Type              1   5275.4 5303.4

Step:  AIC=5298.31
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide + 
    pH + volatile.acidity:Type + Type:total.sulfur.dioxide + 
    density:Type + residual.sugar:Type

                           Df Deviance    AIC
+ pH:Type                   1   5266.7 5296.7
+ chlorides                 1   5268.0 5298.0
<none>                          5270.3 5298.3
+ free.sulfur.dioxide:Type  1   5268.4 5298.4
+ fixed.acidity             1   5269.7 5299.7
+ citric.acid               1   5269.9 5299.9
+ sulphates:Type            1   5270.1 5300.1
+ alcohol:Type              1   5270.2 5300.2

Step:  AIC=5296.67
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide + 
    pH + volatile.acidity:Type + Type:total.sulfur.dioxide + 
    density:Type + residual.sugar:Type + Type:pH

                           Df Deviance    AIC
+ chlorides                 1   5263.4 5295.4
+ free.sulfur.dioxide:Type  1   5263.8 5295.8
<none>                          5266.7 5296.7
+ citric.acid               1   5265.9 5297.9
+ alcohol:Type              1   5266.4 5298.4
+ sulphates:Type            1   5266.6 5298.6
+ fixed.acidity             1   5266.6 5298.6

Step:  AIC=5295.44
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide + 
    pH + chlorides + volatile.acidity:Type + Type:total.sulfur.dioxide + 
    density:Type + residual.sugar:Type + Type:pH

                           Df Deviance    AIC
+ chlorides:Type            1   5259.3 5293.3
+ free.sulfur.dioxide:Type  1   5260.5 5294.5
<none>                          5263.4 5295.4
+ sulphates:Type            1   5262.7 5296.7
+ citric.acid               1   5263.0 5297.0
+ alcohol:Type              1   5263.2 5297.2
+ fixed.acidity             1   5263.4 5297.4

Step:  AIC=5293.31
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide + 
    pH + chlorides + volatile.acidity:Type + Type:total.sulfur.dioxide + 
    density:Type + residual.sugar:Type + Type:pH + Type:chlorides

                           Df Deviance    AIC
+ free.sulfur.dioxide:Type  1   5256.2 5292.2
<none>                          5259.3 5293.3
+ sulphates:Type            1   5257.9 5293.9
+ citric.acid               1   5258.9 5294.9
+ alcohol:Type              1   5259.3 5295.3
+ fixed.acidity             1   5259.3 5295.3

Step:  AIC=5292.16
cat_quality ~ alcohol + volatile.acidity + density + sulphates + 
    residual.sugar + Type + total.sulfur.dioxide + free.sulfur.dioxide + 
    pH + chlorides + volatile.acidity:Type + Type:total.sulfur.dioxide + 
    density:Type + residual.sugar:Type + Type:pH + Type:chlorides + 
    Type:free.sulfur.dioxide

                 Df Deviance    AIC
<none>                5256.2 5292.2
+ sulphates:Type  1   5254.7 5292.7
+ citric.acid     1   5255.9 5293.9
+ alcohol:Type    1   5256.1 5294.1
+ fixed.acidity   1   5256.2 5294.2

Call:  glm(formula = cat_quality ~ alcohol + volatile.acidity + density + 
    sulphates + residual.sugar + Type + total.sulfur.dioxide + 
    free.sulfur.dioxide + pH + chlorides + volatile.acidity:Type + 
    Type:total.sulfur.dioxide + density:Type + residual.sugar:Type + 
    Type:pH + Type:chlorides + Type:free.sulfur.dioxide, family = "binomial", 
    data = train)

Coefficients:
              (Intercept)                    alcohol           volatile.acidity  
                  9.68713                    0.84990                   -2.84978  
                  density                  sulphates             residual.sugar  
                -15.58322                    2.24908                    0.03117  
                     Type       total.sulfur.dioxide        free.sulfur.dioxide  
                195.20143                   -0.01898                    0.02478  
                       pH                  chlorides      volatile.acidity:Type  
                 -0.60200                   -4.02470                   -3.50882  
Type:total.sulfur.dioxide               density:Type        residual.sugar:Type  
                  0.01756                 -201.96697                    0.12808  
                  Type:pH             Type:chlorides   Type:free.sulfur.dioxide  
                  1.50862                    5.10775                   -0.01620  

Degrees of Freedom: 5196 Total (i.e. Null);  5179 Residual
Null Deviance:      6833 
Residual Deviance: 5256     AIC: 5292

The forward step process dropped + sulphates:Type, fixed.acidity, alcohol:Type, and citric.acid By the hierarchical principle, the two non-interactive terms need to be added back because their have interaction terms are in the model.

 model1_int<-glm(formula = cat_quality ~ alcohol + volatile.acidity + density + 
    sulphates + residual.sugar +  Type + total.sulfur.dioxide + 
    free.sulfur.dioxide + pH + chlorides + volatile.acidity:Type + 
    Type:total.sulfur.dioxide + density:Type + residual.sugar:Type + 
    Type:pH + Type:chlorides + Type:free.sulfur.dioxide, family = "binomial", 
    data = train)
summary(model1_int)

Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + density + 
    sulphates + residual.sugar + Type + total.sulfur.dioxide + 
    free.sulfur.dioxide + pH + chlorides + volatile.acidity:Type + 
    Type:total.sulfur.dioxide + density:Type + residual.sugar:Type + 
    Type:pH + Type:chlorides + Type:free.sulfur.dioxide, family = "binomial", 
    data = train)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.2378  -0.8774   0.4148   0.8069   2.4950  

Coefficients:
                            Estimate Std. Error z value Pr(>|z|)    
(Intercept)                9.687e+00  4.539e+01   0.213 0.830982    
alcohol                    8.499e-01  5.665e-02  15.002  < 2e-16 ***
volatile.acidity          -2.850e+00  4.243e-01  -6.716 1.87e-11 ***
density                   -1.558e+01  4.495e+01  -0.347 0.728824    
sulphates                  2.249e+00  3.032e-01   7.418 1.19e-13 ***
residual.sugar             3.117e-02  5.020e-02   0.621 0.534702    
Type                       1.952e+02  5.575e+01   3.501 0.000463 ***
total.sulfur.dioxide      -1.897e-02  2.917e-03  -6.505 7.77e-11 ***
free.sulfur.dioxide        2.478e-02  8.602e-03   2.880 0.003971 ** 
pH                        -6.020e-01  4.860e-01  -1.239 0.215502    
chlorides                 -4.025e+00  1.527e+00  -2.636 0.008391 ** 
volatile.acidity:Type     -3.509e+00  6.200e-01  -5.659 1.52e-08 ***
Type:total.sulfur.dioxide  1.756e-02  3.200e-03   5.487 4.09e-08 ***
density:Type              -2.020e+02  5.584e+01  -3.617 0.000299 ***
residual.sugar:Type        1.281e-01  5.214e-02   2.456 0.014036 *  
Type:pH                    1.509e+00  5.610e-01   2.689 0.007159 ** 
Type:chlorides             5.108e+00  2.438e+00   2.095 0.036133 *  
Type:free.sulfur.dioxide  -1.620e-02  9.147e-03  -1.771 0.076642 .  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 6833.2  on 5196  degrees of freedom
Residual deviance: 5256.2  on 5179  degrees of freedom
AIC: 5292.2

Number of Fisher Scoring iterations: 5

##evaluating model
model1_int_AIC_train <- model1_int$aic
##predicted quality for test data based on training data
preds<-predict(model1_int,newdata=test, type="response")
model1_int_error <- table(test$cat_quality, preds>0.7)
#Curves
evulation_summary_1int <- data.frame(
  attempt = 'model1_int',
  AIC = model1_int_AIC_train,
  PRESS = get_press(model1_int),
  'False positive' = round(model1_int_error[3]/(model1_int_error[1]+model1_int_error[3]),3),
  'False negative' = round(model1_int_error[2]/(model1_int_error[2]+model1_int_error[4]),3),
  'Error Rate' = round((model1_int_error[2]+model1_int_error[3])/(model1_int_error[1]+model1_int_error[2]+model1_int_error[3]+model1_int_error[4]),3)
)

evulation_summary <- rbind(evulation_summary,evulation_summary_1int)
evulation_summary

compare_models<-rbind(compare_models,evulation_summary_1int)
compare_models
NA
compare_models<-compare_models%>% 
  rename(
    Model = attempt
    )
library(data.table)
Registered S3 method overwritten by 'data.table':
  method           from
  print.data.table     
data.table 1.14.0 using 1 threads (see ?getDTthreads).  Latest news: r-datatable.com
**********
This installation of data.table has not detected OpenMP support. It should still work but in single-threaded mode.
This is a Mac. Please read https://mac.r-project.org/openmp/. Please engage with Apple and ask them for support. Check r-datatable.com for updates, and our Mac instructions here: https://github.com/Rdatatable/data.table/wiki/Installation. After several years of many reports of installation problems on Mac, it's time to gingerly point out that there have been no similar problems on Windows or Linux.
**********

Attaching package: ‘data.table’

The following objects are masked from ‘package:reshape2’:

    dcast, melt

The following objects are masked from ‘package:dplyr’:

    between, first, last

The following object is masked from ‘package:purrr’:

    transpose
library(dplyr)
library(formattable)
Registered S3 methods overwritten by 'htmltools':
  method               from         
  print.html           tools:rstudio
  print.shiny.tag      tools:rstudio
  print.shiny.tag.list tools:rstudio
Registered S3 method overwritten by 'htmlwidgets':
  method           from         
  print.htmlwidget tools:rstudio
library(tidyr)
customGreen0 = "#DeF7E9"

customGreen = "#71CA97"

customRed = "#ff7f7f"

Creating the ROC curves and AUC for the 3 new models.

# model1_Red
preds<-predict(model1_Red,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_Red")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
model1_Red_auc <- auc@y.values


# model1_White
preds<-predict(model1_White,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_White")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
model1_White_auc <- auc@y.values


# model1_int
preds<-predict(model1_int,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_int")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
model1_int_auc <- auc@y.values
reduced_1_auc
[[1]]
[1] 0.7949768
model1_Red_auc
[[1]]
[1] 0.7541583
model1_White_auc
[[1]]
[1] 0.7752713
model1_int_auc
[[1]]
[1] 0.8015216

This is the one liners that run the tables and figures!


##create heat map Consolidated
ggplot(data = melted_cor_train, aes(x=Var1, y=Var2, fill=value)) + 
  geom_tile(color = "white")+
  scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
  theme_minimal()+ 
  theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
  coord_fixed()+
  labs(title = 'Consolidated (Both Red and White)')

##create heat map White
ggplot(data = melted_cor_train_white, aes(x=Var1, y=Var2, fill=value)) +
  geom_tile(color = "white")+
  scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
  theme_minimal()+
  theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
  coord_fixed()+
  labs(title = 'White Wine')

##create heat map Red
ggplot(data = melted_cor_train_Red, aes(x=Var1, y=Var2, fill=value)) +
  geom_tile(color = "white")+
  scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = 0, limit = c(-1,1), space = "Lab", name="Pearson\nCorrelation") +
  theme_minimal()+
  theme(axis.text.x = element_text(angle = 45, vjust = 1, size = 12, hjust = 1))+
  coord_fixed()+
  labs(title = 'Red Wine')



ggplot(train_with_qual, mapping = aes(x=quality, fill=Type))+
  geom_histogram(binwidth=1, alpha=.4, position="identity", color="black")+
  geom_vline(aes(xintercept=5.5, color="red"),
             linetype="dashed")+
  scale_color_manual(name = "Cut Off", values = c("red"))+
  labs(x="Quality",
       y="Frequency",
       title="Distribution of Quality Rating by Wine Type")

This is the table for showing the evaluation for the first model

formattable(evulation_summary[1,])
summary(model1_Red)

Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + total.sulfur.dioxide + 
    sulphates + free.sulfur.dioxide + chlorides, family = "binomial", 
    data = train_Red_NoType)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.0655  -0.8628   0.3239   0.8474   2.3082  

Coefficients:
                      Estimate Std. Error z value Pr(>|z|)    
(Intercept)          -8.021553   0.886912  -9.044  < 2e-16 ***
alcohol               0.855123   0.078409  10.906  < 2e-16 ***
volatile.acidity     -2.931148   0.413395  -7.090 1.34e-12 ***
total.sulfur.dioxide -0.018428   0.002926  -6.297 3.03e-10 ***
sulphates             2.637391   0.456973   5.771 7.86e-09 ***
free.sulfur.dioxide   0.023257   0.008608   2.702  0.00689 ** 
chlorides            -4.101411   1.588464  -2.582  0.00982 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 1809.9  on 1308  degrees of freedom
Residual deviance: 1372.6  on 1302  degrees of freedom
AIC: 1386.6

Number of Fisher Scoring iterations: 4
summary(model1_White)

Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + residual.sugar + 
    sulphates + free.sulfur.dioxide + density + pH, family = "binomial", 
    data = train_White_NoType)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.1956  -0.8867   0.4377   0.7961   2.5070  

Coefficients:
                      Estimate Std. Error z value Pr(>|z|)    
(Intercept)          2.105e+02  4.762e+01   4.420 9.87e-06 ***
alcohol              8.408e-01  7.311e-02  11.501  < 2e-16 ***
volatile.acidity    -6.407e+00  4.417e-01 -14.504  < 2e-16 ***
residual.sugar       1.586e-01  1.972e-02   8.045 8.66e-16 ***
sulphates            1.885e+00  3.978e-01   4.739 2.14e-06 ***
free.sulfur.dioxide  6.796e-03  2.536e-03   2.680  0.00737 ** 
density             -2.230e+02  4.773e+01  -4.673 2.97e-06 ***
pH                   9.247e-01  2.859e-01   3.235  0.00122 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 4945.4  on 3887  degrees of freedom
Residual deviance: 3885.3  on 3880  degrees of freedom
AIC: 3901.3

Number of Fisher Scoring iterations: 5
summary(model1_int)

Call:
glm(formula = cat_quality ~ alcohol + volatile.acidity + density + 
    sulphates + residual.sugar + Type + total.sulfur.dioxide + 
    free.sulfur.dioxide + pH + chlorides + volatile.acidity:Type + 
    Type:total.sulfur.dioxide + density:Type + residual.sugar:Type + 
    Type:pH + Type:chlorides + Type:free.sulfur.dioxide, family = "binomial", 
    data = train)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-3.2378  -0.8774   0.4148   0.8069   2.4950  

Coefficients:
                            Estimate Std. Error z value Pr(>|z|)    
(Intercept)                9.687e+00  4.539e+01   0.213 0.830982    
alcohol                    8.499e-01  5.665e-02  15.002  < 2e-16 ***
volatile.acidity          -2.850e+00  4.243e-01  -6.716 1.87e-11 ***
density                   -1.558e+01  4.495e+01  -0.347 0.728824    
sulphates                  2.249e+00  3.032e-01   7.418 1.19e-13 ***
residual.sugar             3.117e-02  5.020e-02   0.621 0.534702    
Type                       1.952e+02  5.575e+01   3.501 0.000463 ***
total.sulfur.dioxide      -1.897e-02  2.917e-03  -6.505 7.77e-11 ***
free.sulfur.dioxide        2.478e-02  8.602e-03   2.880 0.003971 ** 
pH                        -6.020e-01  4.860e-01  -1.239 0.215502    
chlorides                 -4.025e+00  1.527e+00  -2.636 0.008391 ** 
volatile.acidity:Type     -3.509e+00  6.200e-01  -5.659 1.52e-08 ***
Type:total.sulfur.dioxide  1.756e-02  3.200e-03   5.487 4.09e-08 ***
density:Type              -2.020e+02  5.584e+01  -3.617 0.000299 ***
residual.sugar:Type        1.281e-01  5.214e-02   2.456 0.014036 *  
Type:pH                    1.509e+00  5.610e-01   2.689 0.007159 ** 
Type:chlorides             5.108e+00  2.438e+00   2.095 0.036133 *  
Type:free.sulfur.dioxide  -1.620e-02  9.147e-03  -1.771 0.076642 .  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 6833.2  on 5196  degrees of freedom
Residual deviance: 5256.2  on 5179  degrees of freedom
AIC: 5292.2

Number of Fisher Scoring iterations: 5

Table right above the “Best Possible Model (Reduced_4)” section.

formattable(compare_models,align =c("l","c", "c", "c", "c", "r"))

NA

This is the table for showing the best models (top five)

formattable(res.best.logistic$BestModels)

Add ROC for reduced_1, model1_Red, model1_White, model1_int


preds<-predict(reduced_1,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_1")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
reduced_1_auc <- auc@y.values
# model1_Red
preds<-predict(model1_Red,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_Red")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
model1_Red_auc <- auc@y.values

# model1_White
preds<-predict(model1_White,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_White")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
model1_White_auc <- auc@y.values

# model1_int
preds<-predict(model1_int,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for model1_int")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
model1_int_auc <- auc@y.values

This is the table for showing the best models (top five)

formattable(evulation_summary[2,])

This is the table for reduced_4 VIF.

formattable(data.frame(reg_4_VIF_test), align =c("l","r"))

This is the next VIF plot in the report

formattable(data.frame(reg_4_2_VIF_test), align =c("l","r"))

The table below that. It is the evaluation summary for reduced_4_2

formattable(evulation_summary[3,])

evaluation summary for the outlier/leverage/etc.

formattable(evulation_summary[4:7,])

add roc curves for these four.



## reduced_4
preds<-predict(reduced_4,newdata=test, type="response")
rates4<-prediction(preds, test$cat_quality)
roc_result<-performance(rates4,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4")
lines(x = c(0,1), y = c(0,1), col="red")


auc4<-performance(rates4, measure = "auc")
reduced_4_auc <- auc4@y.values

## reduced_4_2
preds<-predict(reduced_4_2,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_2")
lines(x = c(0,1), y = c(0,1), col="red")


auc4_2<-performance(rates, measure = "auc")
reduced_4_2_auc <- auc4_2@y.values

## reduced_4_3
preds<-predict(reduced_4_3,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_3")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
reduced_4_3_auc <- auc@y.values
## reduced_4_4_lev 
preds<-predict(reduced_4_4_lev,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_4_lev")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
reduced_4_4_lev_auc <- auc@y.values

## reduced_4_5_DDFFITS 
preds<-predict(reduced_4_5_DDFFITS,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_5_DDFFITS")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
reduced_4_5_DDFFITS_auc <- auc@y.values

## reduced_4_6_no_special 
preds<-predict(reduced_4_6_no_special,newdata=test, type="response")
rates<-prediction(preds, test$cat_quality)
roc_result<-performance(rates,measure="tpr", x.measure="fpr")
plot(roc_result, main="ROC Curve for reduced_4_6_no_special")
lines(x = c(0,1), y = c(0,1), col="red")


auc<-performance(rates, measure = "auc")
reduced_4_6_no_special_auc <- auc@y.values
AUC_summary <- data.frame('reduced_1'=reduced_1_auc,
                          'reduced_4'=reduced_4_auc,
                          'reduced_4_2'=reduced_4_2_auc,
                          'reduced_4_3'=reduced_4_3_auc,
                          'reduced_4_4_lev'=reduced_4_4_lev_auc,
                          'reduced_4_5_DDFFITS'=reduced_4_5_DDFFITS_auc,
                          'reduced_4_6_no_special'=reduced_4_6_no_special_auc)
colnames(AUC_summary) <- c('reduced_1','reduced_4','reduced_4_2','reduced_4_3','reduced_4_4_lev','reduced_4_5_DDFFITS','reduced_4_6_no_special')

AUC_summary
formattable(evulation_summary[1:7,])

First model and additional EDA

The first research question focused on exploring the relationships between different chemical features in the dataset and wine type (categorized as either red or white). Since the response variable, wine type, was a Bernoulli random variable and the observations were independent, a logistic regression was used for the predictive model.

As seen above in the previous section, exploratory data analysis was performed in order to better understand the relationships between individual predictor variables and wine type. Density plots visualized the quantitative predictors, while frequency bar charts visualized the categorical wine quality predictor. The majority of these features were distributed differently across wine type, indicating that they could be used to differentiate between red and white wine in the model. However, as seen in Figure XXX, alcohol, pH, and sulphates do seem to have a weaker association with wine type since their red and white density plots look somewhat similar.

After exploring these features, an initial model was generated with all of the predictors using the random training sample from the combined wine dataset. From the model summary, the chemical properties of fixed acidity, citric acid, pH, and sulphates all appeared to have insignificant z-scores. To determine if all of these features could be dropped from the model, a likelihood ratio test comparing the full and reduced model was conducted. This test resulted in an insignificant p-value of 0.1360. Therefore, there was not a significant difference between the two models and the reduced model could be adopted. This model’s performance is summarized below. XXX


Call:
glm(formula = Type ~ ., family = "binomial", data = train)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-5.7661  -0.0003   0.0133   0.0465   4.9615  

Coefficients:
                       Estimate Std. Error z value Pr(>|z|)    
(Intercept)           1.525e+03  1.963e+02   7.768 7.98e-15 ***
fixed.acidity         1.287e-01  2.621e-01   0.491  0.62344    
volatile.acidity     -7.722e+00  1.226e+00  -6.300 2.98e-10 ***
citric.acid           1.608e+00  1.295e+00   1.242  0.21428    
residual.sugar        8.972e-01  1.113e-01   8.061 7.58e-16 ***
chlorides            -2.674e+01  5.116e+00  -5.227 1.73e-07 ***
free.sulfur.dioxide  -4.604e-02  1.679e-02  -2.742  0.00611 ** 
total.sulfur.dioxide  5.215e-02  5.436e-03   9.594  < 2e-16 ***
density              -1.518e+03  2.007e+02  -7.564 3.89e-14 ***
pH                    3.214e-03  1.620e+00   0.002  0.99842    
sulphates            -4.389e+00  1.383e+00  -3.174  0.00150 ** 
alcohol              -1.188e+00  2.915e-01  -4.077 4.57e-05 ***
quality              -4.123e-01  2.294e-01  -1.797  0.07235 .  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 5866.24  on 5196  degrees of freedom
Residual deviance:  337.27  on 5184  degrees of freedom
AIC: 363.27

Number of Fisher Scoring iterations: 10

[[1]]
[1] 0.9946774
       
        FALSE TRUE
  Red     285    5
  White     3 1007

Call:
glm(formula = Type ~ . - quality - pH - fixed.acidity - citric.acid, 
    family = "binomial", data = train)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-5.8141  -0.0002   0.0125   0.0439   4.4116  

Coefficients:
                       Estimate Std. Error z value Pr(>|z|)    
(Intercept)           1.430e+03  1.226e+02  11.664  < 2e-16 ***
volatile.acidity     -8.193e+00  1.007e+00  -8.139 4.00e-16 ***
residual.sugar        8.752e-01  9.663e-02   9.058  < 2e-16 ***
chlorides            -2.542e+01  4.340e+00  -5.857 4.71e-09 ***
free.sulfur.dioxide  -5.730e-02  1.525e-02  -3.757 0.000172 ***
total.sulfur.dioxide  5.588e-02  5.210e-03  10.725  < 2e-16 ***
density              -1.423e+03  1.219e+02 -11.672  < 2e-16 ***
sulphates            -5.093e+00  1.360e+00  -3.744 0.000181 ***
alcohol              -1.225e+00  2.323e-01  -5.274 1.34e-07 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 5866.24  on 5196  degrees of freedom
Residual deviance:  344.26  on 5188  degrees of freedom
AIC: 362.26

Number of Fisher Scoring iterations: 9

The ROC curve XXX, seen below, indicates that the model performed well (significantly better than random guessing shown by the red line). Furthermore, the AUC, which represents the area under this curve, was 0.994. These performance metrics suggest that the model is quite effective at predicting the wine type of the training data. When this model predicted the wine type of the testing data (the sample leftover from the training set), it maintained this high performance. We used a probability floor of 0.65 since this value minimized overall error (we saw no practical difference between type I and type II errors in this context). Using this value, the model had an overall error rate of only 0.0062. This result demonstrates the effectiveness of the model when presented with data it had not encountered in training.

       
        FALSE TRUE
  Red     285    5
  White     3 1007

To further improve the model’s performance, the presence of outliers in the training data was considered. Using Cook’s distance, one outlier (with a distance of 3.0858) was identified. However, after refitting a model without this outlier, there was a very small difference in overall performance as seen below XXX. Thus, we elected to keep the outlier in the data—especially, since we had little insight into the data and could not deduce whether it was a corrupted datapoint.


Call:
glm(formula = Type ~ . - quality - sulphates - citric.acid, family = "binomial", 
    data = new_data)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-5.6180  -0.0002   0.0167   0.0427   4.5510  

Coefficients:
                       Estimate Std. Error z value Pr(>|z|)    
(Intercept)           2.468e+03  2.139e+02  11.534  < 2e-16 ***
fixed.acidity         1.046e+00  2.562e-01   4.083 4.44e-05 ***
volatile.acidity     -6.460e+00  1.050e+00  -6.155 7.51e-10 ***
residual.sugar        1.016e+00  1.039e-01   9.783  < 2e-16 ***
chlorides            -2.449e+01  4.777e+00  -5.126 2.96e-07 ***
free.sulfur.dioxide  -5.566e-02  1.710e-02  -3.255  0.00113 ** 
total.sulfur.dioxide  5.676e-02  5.716e-03   9.930  < 2e-16 ***
density              -2.476e+03  2.177e+02 -11.374  < 2e-16 ***
pH                    3.867e+00  1.647e+00   2.347  0.01890 *  
alcohol              -2.650e+00  3.304e-01  -8.021 1.05e-15 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 5865.66  on 5195  degrees of freedom
Residual deviance:  286.49  on 5186  degrees of freedom
AIC: 306.49

Number of Fisher Scoring iterations: 9

       
        FALSE TRUE
  Red     285    5
  White     2 1008

Finally, collinearity among the predictors was examined in the reduced model. Using the VIF formula, density was very correlated with the other predictors (having a VIF value of 13.714). Since this value was over 10, this parameter was dropped in order to reduce collinearity and by extension the predicted coefficients’ variances. In addition, the correlation matrix of the predictors showed a strong association between free sulfur dioxide and total sulfur dioxide (correlation of 0.7156). Since one of these features could explain much of the variation in the other, it was determined one should be dropped. When comparing models with only one of these features, it was evident that total sulfur dioxide was more important in predicting wine type than free sulfur dioxide. Therefore, free sulfur dioxide was dropped from the model. The performance of this new reduced model can be seen below XXX.

       fixed.acidity     volatile.acidity       residual.sugar            chlorides 
            4.534992             1.648202             6.411491             1.539680 
 free.sulfur.dioxide total.sulfur.dioxide              density                   pH 
            2.106400             2.801104            13.714057             2.502181 
             alcohol 
            4.153380 
                     fixed.acidity volatile.acidity residual.sugar   chlorides free.sulfur.dioxide
fixed.acidity           1.00000000       0.21475470     -0.1143578  0.30907486         -0.28545891
volatile.acidity        0.21475470       1.00000000     -0.1971152  0.38572698         -0.36296634
residual.sugar         -0.11435784      -0.19711519      1.0000000 -0.13267263          0.40674512
chlorides               0.30907486       0.38572698     -0.1326726  1.00000000         -0.20583891
free.sulfur.dioxide    -0.28545891      -0.36296634      0.4067451 -0.20583891          1.00000000
total.sulfur.dioxide   -0.32829667      -0.42380894      0.4945946 -0.29216595          0.71557904
density                 0.46102677       0.27340816      0.5468642  0.36564000          0.01712365
pH                     -0.25053297       0.26430327     -0.2656167  0.04191555         -0.15788076
alcohol                -0.09179407      -0.03401873     -0.3484276 -0.25617404         -0.17708863
                     total.sulfur.dioxide     density          pH     alcohol
fixed.acidity                  -0.3282967  0.46102677 -0.25053297 -0.09179407
volatile.acidity               -0.4238089  0.27340816  0.26430327 -0.03401873
residual.sugar                  0.4945946  0.54686423 -0.26561668 -0.34842765
chlorides                      -0.2921660  0.36564000  0.04191555 -0.25617404
free.sulfur.dioxide             0.7155790  0.01712365 -0.15788076 -0.17708863
total.sulfur.dioxide            1.0000000  0.02373810 -0.24648617 -0.26082628
density                         0.0237381  1.00000000  0.01687243 -0.67847127
pH                             -0.2464862  0.01687243  1.00000000  0.11700658
alcohol                        -0.2608263 -0.67847127  0.11700658  1.00000000
glm.fit: fitted probabilities numerically 0 or 1 occurred

Call:
glm(formula = Type ~ . - quality - pH - fixed.acidity - citric.acid - 
    density - free.sulfur.dioxide, family = "binomial", data = train)

Deviance Residuals: 
    Min       1Q   Median       3Q      Max  
-6.1013   0.0000   0.0139   0.0522   3.0217  

Coefficients:
                       Estimate Std. Error z value Pr(>|z|)    
(Intercept)            1.073845   1.177468   0.912    0.362    
volatile.acidity     -12.420039   0.793575 -15.651  < 2e-16 ***
residual.sugar         0.190465   0.047838   3.981 6.85e-05 ***
chlorides            -36.881169   3.376694 -10.922  < 2e-16 ***
total.sulfur.dioxide   0.065168   0.003563  18.292  < 2e-16 ***
sulphates            -12.241130   0.880881 -13.896  < 2e-16 ***
alcohol                0.726637   0.100961   7.197 6.14e-13 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 5866.24  on 5196  degrees of freedom
Residual deviance:  644.11  on 5190  degrees of freedom
AIC: 658.11

Number of Fisher Scoring iterations: 9

       
        FALSE TRUE
  Red     284    6
  White    25  985

Though the model’s performance decreased slightly (overall error rate went from 0.0062 to 0.0297 and AUC went from 0.9947 to 0.9892), the decrease in variance of the coefficients should make the model more reliable in terms of interpreting the relationships between the predictors and wine type.

Overall, this final model is quite effective at determining wine type. When looking at the estimated coefficients, it seems as though higher residual sugar, total sulfur dioxide, and alcohol are associated with white wines, while higher volatile acidity, chlorides, and sulphates are associated with red wines. Together, these predictors explain much of the variation in the predictor variable, wine type (residual deviance of only 286.49).

LS0tCnRpdGxlOiAiUiBOb3RlYm9vayIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKCgojIHB1bGxpbmcgaW4gREZzIGFuZCBNZXJnaW5nIChzdGlsbCBuZWVkZWQ/KQpgYGB7cn0KIyAjcmVkIHdpbmVzCiMgUmVkX3dpbmUgPC0gcmVhZC5jc3YoIndpbmVRdWFsaXR5UmVkcy5jc3YiLCBoZWFkZXI9VFJVRSwgc2VwID0gIiwiKQojIFJlZF93aW5lJFR5cGUgPC0gJ1JlZCcKIyAKIyAjd2hpdGUgd2luZXMKIyBXaGl0ZV93aW5lIDwtIHJlYWQuY3N2KCJ3aW5lUXVhbGl0eVdoaXRlcy5jc3YiLCBoZWFkZXI9VFJVRSwgc2VwID0gIiwiKQojIFdoaXRlX3dpbmUkVHlwZSA8LSAnV2hpdGUnCiMgCiMgIyMgY29uc29saWRhdGVkCiMgRGF0YSA8LSByYmluZChSZWRfd2luZSxXaGl0ZV93aW5lKQojIGRyb3BzIDwtIGMoIlgiKQojIERhdGEgPC0gRGF0YVsgLCAhKG5hbWVzKERhdGEpICVpbiUgZHJvcHMpXQojIERhdGEKIyAKIyAjI2NyZWF0ZSBmaW5hbCBERgojICMgd3JpdGUuY3N2KERhdGEsIi9Vc2Vycy9jb2xpbm9icmllbi9EZXNrdG9wL3JlcG8vc3RhdHNfNjAyMS9TdGF0c19wcm9qZWN0X2dyb3VwXzYvRGF0YS5jc3YiLCByb3cubmFtZXMgPSBGQUxTRSkKIyAjIHdyaXRlLmNzdihEYXRhLCIvVXNlcnMvY29saW5vYnJpZW4vRGVza3RvcC9yZXBvL3N0YXRzXzYwMjEvU3RhdHNfcHJvamVjdF9ncm91cF82L0RhdGEiLCByb3cubmFtZXMgPSBGQUxTRSkKIyAjIyBib3RoIHRoZSBEYXRhIGFuZCBEYXRhIGNzdiBhcmUgdGhlIHNhbWUuIEkga25vdyBwZW9wbGUgcHJlZmVyIG9uZSBmb3JtYXQgdnMgdGhlIG90aGVyIHNvIEkgbWFkZSBib3RoCgpgYGAKCgoKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKIyBsaWJyYXJ5KFJPQ1IpCmxpYnJhcnkoZmFyYXdheSkKbGlicmFyeShkcGx5cikKbGlicmFyeShnZ3Bsb3QyKQpsaWJyYXJ5KHJlc2hhcGUyKQpsaWJyYXJ5KGxlYXBzKQojIGluc3RhbGwucGFja2FnZXMoImJlc3RnbG0iKQpsaWJyYXJ5KGJlc3RnbG0pCiMgaW5zdGFsbC5wYWNrYWdlcygicGVyZm9ybWFuY2UiKQojIGxpYnJhcnkocGVyZm9ybWFuY2UpCmtuaXRyOjpvcHRzX2NodW5rJHNldChlY2hvID0gVFJVRSkKCgoKIyMgTG9hZCBEYXRhc2V0cwpmdWxsX3dpbmVzX2ZpbmFsIDwtIHJlYWQuY3N2KCJEYXRhX0ZpbmFsLmNzdiIsIGhlYWRlciA9IFRSVUUsIHN0cmluZ3NBc0ZhY3RvcnM9VFJVRSkKIyBEcm9wIHF1YWxpdHkgZm9yIHNpbXBsaWNpdHkKZnVsbF93aW5lc19iaW5hcnlfd2l0aF9xdWFsPC1mdWxsX3dpbmVzX2ZpbmFsCmZ1bGxfd2luZXNfYmluYXJ5IDwtIHN1YnNldChmdWxsX3dpbmVzX2ZpbmFsLCBzZWxlY3QgPSAtYyhxdWFsaXR5KSkKIyMgQ29udmVydCB0byAwIGFuZCAxIGZvciByZWFkYWJpbGl0eQpmdWxsX3dpbmVzX2JpbmFyeSRjYXRfcXVhbGl0eSA8LSBhcy5pbnRlZ2VyKGZ1bGxfd2luZXNfYmluYXJ5JGNhdF9xdWFsaXR5ID09ICJIaWdoIikKCnNldC5zZWVkKDkwMjEwKSAjI2ZvciByZXByb2R1Y2liaWxpdHkKc2FtcGxlPC1zYW1wbGUuaW50KG5yb3coZnVsbF93aW5lc19iaW5hcnkpLCBmbG9vciguODAqbnJvdyhmdWxsX3dpbmVzX2JpbmFyeSkpLCByZXBsYWNlID0gRikKdHJhaW48LWZ1bGxfd2luZXNfYmluYXJ5W3NhbXBsZSwgXSAjI3RyYWluaW5nIGRhdGEgZnJhbWUKcm93bmFtZXModHJhaW4pIDwtIGMoMTo1MTk3KQp0ZXN0PC1mdWxsX3dpbmVzX2JpbmFyeVstc2FtcGxlLCBdICMjdGVzdCBkYXRhIGZyYW1lCgojIyBKdXN0IGZvciBhIHNpbmdsZSBib3hwbG90CnRyYWluX3dpdGhfcXVhbDwtZnVsbF93aW5lc19iaW5hcnlfd2l0aF9xdWFsW3NhbXBsZSxdCnRlc3Rfd2l0aF9xdWFsPC1mdWxsX3dpbmVzX2JpbmFyeV93aXRoX3F1YWxbLXNhbXBsZSxdCgoKdHJhaW4KYGBgCiMgRURBCgoKYGBge3J9CiMgZHJvcHNfY2F0cyA8LSBjKCJUeXBlIikKIyBOb19jYXRfdHJhaW4gPC0gdHJhaW5bICwgIShuYW1lcyh0cmFpbikgJWluJSBkcm9wc19jYXRzKV0KIyAjIE5vX1R5cGUKCnBhaXJzKHRyYWluLCBsb3dlci5wYW5lbCA9IE5VTEwpCmBgYAoKYGBge3J9CiMgQ29udmVydCBUeXBlIHRvIGJpbmFyeSB0byAwIGFuZCAxIGZvciBjb3JyZWxhdGlvbgp0cmFpbiRUeXBlIDwtIGFzLmludGVnZXIodHJhaW4kVHlwZSA9PSAiV2hpdGUiKQp0ZXN0JFR5cGUgPC0gYXMuaW50ZWdlcih0ZXN0JFR5cGUgPT0gIldoaXRlIikKY29yX3RyYWluIDwtIGNvcih0cmFpbikKY29yX3RyYWluCmBgYAoKCgpgYGB7cn0KVF9GX2NvciA8LSBhYnMoY29yX3RyYWluKT4uNwpUX0ZfY29yCmBgYApgYGB7cn0KIyMgY3JlYXRlIG1lbHRlZAptZWx0ZWRfY29yX3RyYWluIDwtIG1lbHQoY29yX3RyYWluKQoKIyNjcmVhdGUgaGVhdCBtYXAgQ29uc29saWRhdGVkCmdncGxvdChkYXRhID0gbWVsdGVkX2Nvcl90cmFpbiwgYWVzKHg9VmFyMSwgeT1WYXIyLCBmaWxsPXZhbHVlKSkgKyAKICBnZW9tX3RpbGUoY29sb3IgPSAid2hpdGUiKSsKICBzY2FsZV9maWxsX2dyYWRpZW50Mihsb3cgPSAiYmx1ZSIsIGhpZ2ggPSAicmVkIiwgbWlkID0gIndoaXRlIiwgbWlkcG9pbnQgPSAwLCBsaW1pdCA9IGMoLTEsMSksIHNwYWNlID0gIkxhYiIsIG5hbWU9IlBlYXJzb25cbkNvcnJlbGF0aW9uIikgKwogIHRoZW1lX21pbmltYWwoKSsgCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgdmp1c3QgPSAxLCBzaXplID0gMTIsIGhqdXN0ID0gMSkpKwogIGNvb3JkX2ZpeGVkKCkrCiAgbGFicyh0aXRsZSA9ICdDb25zb2xpZGF0ZWQgKEJvdGggUmVkIGFuZCBXaGl0ZSknKQoKCgoKYGBgCgpgYGB7cn0KIyMgY3JlYXRpbmcgcmVkIGFuZCB3aGl0ZQp0cmFpbl9XaGl0ZSA8LSBmaWx0ZXIodHJhaW4sIFR5cGUgPT0gMSkKdHJhaW5fUmVkIDwtIGZpbHRlcih0cmFpbiwgVHlwZSA9PSAwKQoKCiMjIGRyb3BpbmcgcmVkL3doaXRlIGNvbHVtbnMKdHJhaW5fV2hpdGVfTm9UeXBlIDwtIHN1YnNldCh0cmFpbl9XaGl0ZSwgc2VsZWN0ID0gLWMoVHlwZSkpCnRyYWluX1JlZF9Ob1R5cGUgPC0gc3Vic2V0KHRyYWluX1JlZCwgc2VsZWN0ID0gLWMoVHlwZSkpCgojIyBjcmVhdGluZyBjb3JyZWxhdGlvbnMKY29yX3RyYWluX1doaXRlX05vVHlwZSA8LSBjb3IodHJhaW5fV2hpdGVfTm9UeXBlKQpjb3JfdHJhaW5fUmVkX05vVHlwZSA8LSBjb3IodHJhaW5fUmVkX05vVHlwZSkKCiMjIG1lbHRpbmcKbWVsdGVkX2Nvcl90cmFpbl93aGl0ZSA8LSBtZWx0KGNvcl90cmFpbl9XaGl0ZV9Ob1R5cGUpCm1lbHRlZF9jb3JfdHJhaW5fUmVkIDwtIG1lbHQoY29yX3RyYWluX1JlZF9Ob1R5cGUpCgojI3Bsb3RpbmcKCiMjY3JlYXRlIGhlYXQgbWFwIFdoaXRlCmdncGxvdChkYXRhID0gbWVsdGVkX2Nvcl90cmFpbl93aGl0ZSwgYWVzKHg9VmFyMSwgeT1WYXIyLCBmaWxsPXZhbHVlKSkgKwogIGdlb21fdGlsZShjb2xvciA9ICJ3aGl0ZSIpKwogIHNjYWxlX2ZpbGxfZ3JhZGllbnQyKGxvdyA9ICJibHVlIiwgaGlnaCA9ICJyZWQiLCBtaWQgPSAid2hpdGUiLCBtaWRwb2ludCA9IDAsIGxpbWl0ID0gYygtMSwxKSwgc3BhY2UgPSAiTGFiIiwgbmFtZT0iUGVhcnNvblxuQ29ycmVsYXRpb24iKSArCiAgdGhlbWVfbWluaW1hbCgpKwogIHRoZW1lKGF4aXMudGV4dC54ID0gZWxlbWVudF90ZXh0KGFuZ2xlID0gNDUsIHZqdXN0ID0gMSwgc2l6ZSA9IDEyLCBoanVzdCA9IDEpKSsKICBjb29yZF9maXhlZCgpKwogIGxhYnModGl0bGUgPSAnV2hpdGUgV2luZScpCgojI2NyZWF0ZSBoZWF0IG1hcCBSZWQKZ2dwbG90KGRhdGEgPSBtZWx0ZWRfY29yX3RyYWluX1JlZCwgYWVzKHg9VmFyMSwgeT1WYXIyLCBmaWxsPXZhbHVlKSkgKwogIGdlb21fdGlsZShjb2xvciA9ICJ3aGl0ZSIpKwogIHNjYWxlX2ZpbGxfZ3JhZGllbnQyKGxvdyA9ICJibHVlIiwgaGlnaCA9ICJyZWQiLCBtaWQgPSAid2hpdGUiLCBtaWRwb2ludCA9IDAsIGxpbWl0ID0gYygtMSwxKSwgc3BhY2UgPSAiTGFiIiwgbmFtZT0iUGVhcnNvblxuQ29ycmVsYXRpb24iKSArCiAgdGhlbWVfbWluaW1hbCgpKwogIHRoZW1lKGF4aXMudGV4dC54ID0gZWxlbWVudF90ZXh0KGFuZ2xlID0gNDUsIHZqdXN0ID0gMSwgc2l6ZSA9IDEyLCBoanVzdCA9IDEpKSsKICBjb29yZF9maXhlZCgpKwogIGxhYnModGl0bGUgPSAnUmVkIFdpbmUnKQoKYGBgCgpgYGB7cn0KZ2dwbG90KGRhdGEgPSB0cmFpbiwgbWFwcGluZyA9IGFlcyh4PVR5cGUpKSArIAogIGdlb21fYmFyKCkKYGBgCgojIFJlZ3Jlc3Npb24gVGVzdGluZwoKYGBge3J9CiMjIHByZXNzIGZvcm11bGEgKGZyb20gY2xhc3MpCmdldF9wcmVzcyA8LSBmdW5jdGlvbihtb2RlbCkgewogIHN1bSgoKG1vZGVsJHJlc2lkdWFscykvICgxLSAobG0uaW5mbHVlbmNlKG1vZGVsKSRoYXQpKSleMikKfQpgYGAKCgpgYGB7cn0KIyMgZmlyc3QgZ28KZnVsbDwtZ2xtKGNhdF9xdWFsaXR5fi4sIGZhbWlseT1iaW5vbWlhbCwgZGF0YT10cmFpbikKc3VtbWFyeShmdWxsKQpgYGAKYGBge3J9CiMjIHJlbW92ZWQgYWxsIGluc2lnbmlmaWNhbnQKcmVkdWNlZF8xPC1nbG0oZm9ybXVsYSA9IGNhdF9xdWFsaXR5fnZvbGF0aWxlLmFjaWRpdHkrcmVzaWR1YWwuc3VnYXIrZnJlZS5zdWxmdXIuZGlveGlkZSt0b3RhbC5zdWxmdXIuZGlveGlkZStkZW5zaXR5K3BIK3N1bHBoYXRlcythbGNvaG9sK1R5cGUsIGZhbWlseT1iaW5vbWlhbCwgZGF0YT10cmFpbikKc3VtbWFyeShyZWR1Y2VkXzEpCmBgYAoKYGBge3J9CiMjZXZhbHVhdGluZyBtb2RlbApSZWR1Y2VkMV9BSUNfdHJhaW4gPC0gcmVkdWNlZF8xJGFpYwoKIyNwcmVkaWN0ZWQgcXVhbGl0eSBmb3IgdGVzdCBkYXRhIGJhc2VkIG9uIHRyYWluaW5nIGRhdGEKcHJlZHM8LXByZWRpY3QocmVkdWNlZF8xLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQoKcmVkdWNlZF8xX2Vycm9yIDwtIHRhYmxlKHRlc3QkY2F0X3F1YWxpdHksIHByZWRzPjAuNSkKCnJlZHVjZWRfMV9lcnJvcgoKZXZ1bGF0aW9uX3N1bW1hcnkgPC0gZGF0YS5mcmFtZSgKICBhdHRlbXB0ID0gJ3JlZHVjZWRfMScsCiAgQUlDID0gUmVkdWNlZDFfQUlDX3RyYWluLAogIFBSRVNTID0gZ2V0X3ByZXNzKHJlZHVjZWRfMSksCiAgJ0ZhbHNlIHBvc2l0aXZlJyA9IHJvdW5kKHJlZHVjZWRfMV9lcnJvclszXS8ocmVkdWNlZF8xX2Vycm9yWzFdK3JlZHVjZWRfMV9lcnJvclszXSksMyksCiAgJ0ZhbHNlIG5lZ2F0aXZlJyA9IHJvdW5kKHJlZHVjZWRfMV9lcnJvclsyXS8ocmVkdWNlZF8xX2Vycm9yWzJdK3JlZHVjZWRfMV9lcnJvcls0XSksMyksCiAgJ0Vycm9yIFJhdGUnID0gcm91bmQoKHJlZHVjZWRfMV9lcnJvclsyXStyZWR1Y2VkXzFfZXJyb3JbM10pLyhyZWR1Y2VkXzFfZXJyb3JbMV0rcmVkdWNlZF8xX2Vycm9yWzJdK3JlZHVjZWRfMV9lcnJvclszXStyZWR1Y2VkXzFfZXJyb3JbNF0pLDMpCikKZXZ1bGF0aW9uX3N1bW1hcnkKYGBgCgojIyBzZWNvbmQgbW9kZWwKIyMgaHR0cHM6Ly9yc3R1ZGlvLXB1YnMtc3RhdGljLnMzLmFtYXpvbmF3cy5jb20vMjg5N185MjIwYjIxY2ZjMGM0M2EzOTZmZjlhYmYxMjJiYjM1MS5odG1sCgpgYGB7cn0KIyBpbnN0YWxsLnBhY2thZ2VzKCJiZXN0Z2xtIikKIyMgUHJlcGFyZSBkYXRhCnRyYWluLmZvci5iZXN0LmxvZ2lzdGljIDwtIHdpdGhpbih0cmFpbiwgewogICAgeSA8LSBjYXRfcXVhbGl0eSAKfSkKCiMjIFJlb3JkZXIgdmFyaWFibGVzCnRyYWluLmZvci5iZXN0LmxvZ2lzdGljIDwtCiAgICB0cmFpbi5mb3IuYmVzdC5sb2dpc3RpY1ssIGMoImZpeGVkLmFjaWRpdHkiLCJ2b2xhdGlsZS5hY2lkaXR5IiwiY2l0cmljLmFjaWQiLCJyZXNpZHVhbC5zdWdhciIsInRvdGFsLnN1bGZ1ci5kaW94aWRlIiwiZGVuc2l0eSIsImNobG9yaWRlcyIsImZyZWUuc3VsZnVyLmRpb3hpZGUiLCdwSCcsJ3N1bHBoYXRlcycsJ2FsY29ob2wnLCdUeXBlJywieSIpXQoKIyMgUGVyZm9ybQpyZXMuYmVzdC5sb2dpc3RpYyA8LQogICAgYmVzdGdsbShYeSA9IHRyYWluLmZvci5iZXN0LmxvZ2lzdGljLAogICAgICAgICAgICBmYW1pbHkgPSBiaW5vbWlhbCwgICAgICAgICAgIyBiaW5vbWlhbCBmYW1pbHkgZm9yIGxvZ2lzdGljCiAgICAgICAgICAgIElDID0gIkFJQyIsICAgICAgICAgICAgICAgICAjIEluZm9ybWF0aW9uIGNyaXRlcmlhIGZvcgogICAgICAgICAgICBtZXRob2QgPSAiZXhoYXVzdGl2ZSIpCmBgYAoKCmBgYHtyfQpyZXMuYmVzdC5sb2dpc3RpYyRCZXN0TW9kZWxzCnN1bW1hcnkocmVzLmJlc3QubG9naXN0aWMkQmVzdE1vZGVsKQpgYGAKYGBge3J9CnJlZHVjZWRfNCA8LSByZXMuYmVzdC5sb2dpc3RpYyRCZXN0TW9kZWwKIyNldmFsdWF0aW5nIG1vZGVsClJlZHVjZWQ0X0FJQ190cmFpbiA8LSByZWR1Y2VkXzQkYWljCgojI3ByZWRpY3RlZCBxdWFsaXR5IGZvciB0ZXN0IGRhdGEgYmFzZWQgb24gdHJhaW5pbmcgZGF0YQpwcmVkczwtcHJlZGljdChyZWR1Y2VkXzQsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCgpyZWR1Y2VkXzRfZXJyb3IgPC0gdGFibGUodGVzdCRjYXRfcXVhbGl0eSwgcHJlZHM+MC41KQoKZXZ1bGF0aW9uX3N1bW1hcnlfNCA8LSBkYXRhLmZyYW1lKAogIGF0dGVtcHQgPSAncmVkdWNlZF80X2Vycm9yIChhbGwgcG9zc2libGUpJywKICBBSUMgPSBSZWR1Y2VkNF9BSUNfdHJhaW4sCiAgUFJFU1MgPSBnZXRfcHJlc3MocmVkdWNlZF80KSwKICAnRmFsc2UgcG9zaXRpdmUnID0gcm91bmQocmVkdWNlZF80X2Vycm9yWzNdLyhyZWR1Y2VkXzRfZXJyb3JbMV0rcmVkdWNlZF80X2Vycm9yWzNdKSwzKSwKICAnRmFsc2UgbmVnYXRpdmUnID0gcm91bmQocmVkdWNlZF80X2Vycm9yWzJdLyhyZWR1Y2VkXzRfZXJyb3JbMl0rcmVkdWNlZF80X2Vycm9yWzRdKSwzKSwKICAnRXJyb3IgUmF0ZScgPSByb3VuZCgocmVkdWNlZF80X2Vycm9yWzJdK3JlZHVjZWRfNF9lcnJvclszXSkvKHJlZHVjZWRfNF9lcnJvclsxXStyZWR1Y2VkXzRfZXJyb3JbMl0rcmVkdWNlZF80X2Vycm9yWzNdK3JlZHVjZWRfNF9lcnJvcls0XSksMykKKQoKZXZ1bGF0aW9uX3N1bW1hcnkgPC0gcmJpbmQoZXZ1bGF0aW9uX3N1bW1hcnksZXZ1bGF0aW9uX3N1bW1hcnlfNCkKIyBldnVsYXRpb25fc3VtbWFyeQpgYGAKCmBgYHtyfQojIGRhdGEuZnJhbWUoY2hlY2tfY29sbGluZWFyaXR5KHJlZHVjZWRfNCkpCgoKCgoKCgoKCgojY29tZSBiYWNrIGFuZCBhZGQgZGYgc3R1ZmYKYGBgCgojIyBpbiBhbiBlZmZvcnQgdG8gbG93ZXIgVklGcyBzY29yZXMgYW5kIGNvcnJlbGF0aW9uLCBJIGFtIHJlbW92aW5nIGZpeGVkLmFjaWRpdHkKCmBgYHtyfQpyZWR1Y2VkXzRfMjwtZ2xtKGNhdF9xdWFsaXR5fnZvbGF0aWxlLmFjaWRpdHkrY2l0cmljLmFjaWQrcmVzaWR1YWwuc3VnYXIrdG90YWwuc3VsZnVyLmRpb3hpZGUrZGVuc2l0eStmcmVlLnN1bGZ1ci5kaW94aWRlK3BIK3N1bHBoYXRlcythbGNvaG9sK1R5cGUsIGZhbWlseT1iaW5vbWlhbCwgZGF0YT10cmFpbikKc3VtbWFyeShyZWR1Y2VkXzRfMikKYGBgCmBgYHtyfQojI2V2YWx1YXRpbmcgbW9kZWwKUmVkdWNlZDRfMl9BSUNfdHJhaW4gPC0gcmVkdWNlZF80XzIkYWljCiMjcHJlZGljdGVkIHF1YWxpdHkgZm9yIHRlc3QgZGF0YSBiYXNlZCBvbiB0cmFpbmluZyBkYXRhCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF8yLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyZWR1Y2VkXzRfMl9lcnJvciA8LSB0YWJsZSh0ZXN0JGNhdF9xdWFsaXR5LCBwcmVkcz4wLjcpCiNDdXJ2ZXMKZXZ1bGF0aW9uX3N1bW1hcnlfNF8yIDwtIGRhdGEuZnJhbWUoCiAgYXR0ZW1wdCA9ICdyZWR1Y2VkXzRfMl9lcnJvciAocG9zdCBWSUYgYWRqdXN0bWVudHMpJywKICBBSUMgPSBSZWR1Y2VkNF8yX0FJQ190cmFpbiwKICBQUkVTUyA9IGdldF9wcmVzcyhyZWR1Y2VkXzRfMiksCiAgJ0ZhbHNlIHBvc2l0aXZlJyA9IHJvdW5kKHJlZHVjZWRfNF8yX2Vycm9yWzNdLyhyZWR1Y2VkXzRfMl9lcnJvclsxXStyZWR1Y2VkXzRfMl9lcnJvclszXSksMyksCiAgJ0ZhbHNlIG5lZ2F0aXZlJyA9IHJvdW5kKHJlZHVjZWRfNF8yX2Vycm9yWzJdLyhyZWR1Y2VkXzRfMl9lcnJvclsyXStyZWR1Y2VkXzRfMl9lcnJvcls0XSksMyksCiAgJ0Vycm9yIFJhdGUnID0gcm91bmQoKHJlZHVjZWRfNF8yX2Vycm9yWzJdK3JlZHVjZWRfNF8yX2Vycm9yWzNdKS8ocmVkdWNlZF80XzJfZXJyb3JbMV0rcmVkdWNlZF80XzJfZXJyb3JbMl0rcmVkdWNlZF80XzJfZXJyb3JbM10rcmVkdWNlZF80XzJfZXJyb3JbNF0pLDMpCikKCmV2dWxhdGlvbl9zdW1tYXJ5IDwtIHJiaW5kKGV2dWxhdGlvbl9zdW1tYXJ5LGV2dWxhdGlvbl9zdW1tYXJ5XzRfMikKZXZ1bGF0aW9uX3N1bW1hcnkKCgpgYGAKCiMjIG5vdyBsb29raW5nIGF0IG91dGxpZXJzICh3aXRoICJiZXN0IHBvc3NpYmxlIikKCmBgYHtyfQpzdW1tYXJ5KHJlZHVjZWRfNCkKCmBgYAoKCiMgTm93IGxvb2tpbmcgaW50byBvdXRsaWVycy9pbmZsdWVuY2UKCmBgYHtyfQpwIDwtIDEyCm4gPC0gNTE5NwpgYGAKCiMjIyBDb29rcwpgYGB7cn0KcmVkdWNlZF80X2Nvb2sgPC1jb29rcy5kaXN0YW5jZShyZWR1Y2VkXzQpCnJlZHVjZWRfNF9jb29rW3JlZHVjZWRfNF9jb29rPnFmKDAuNSxwLG4tcCldCmBgYAojIyMgREZGSVRzCgpgYGB7cn0KIyNkZmZpdHMKREZGSVRTPC1kZmZpdHMocmVkdWNlZF80KQpEREZGSVRTX2luZmx1ZW5jZSA8LSBERkZJVFNbYWJzKERGRklUUyk+MipzcXJ0KHAvbildCkRERkZJVFNfaW5mbHVlbmNlCmBgYAojIyMgREZCRVRBcwpgYGB7cn0KREZCRVRBUzwtZGZiZXRhcyhyZWR1Y2VkXzQpCmFicyhERkJFVEFTKT4yL3NxcnQobikKYGBgCgoKIyMjIGxldmVyYWdlCmBgYHtyfQojI2xldmVyYWdlcwpsZXY8LWxtLmluZmx1ZW5jZShyZWR1Y2VkXzQpJGhhdAojI2lkZW50aWZ5IGhpZ2ggbGV2ZXJhZ2UgcG9pbnRzCmxldmVyYWdlcyA8LSBsZXZbbGV2PjIqcC9uXQpsZXZlcmFnZXMKYGBgCgoKIyMjIG91dGxpZXIKCmBgYHtyfQpyZWR1Y2VkXzQucmVzIDwtIHJlZHVjZWRfNCRyZXNpZHVhbHMKY3JpdDwtcXQoMS0wLjA1LygyKm4pLCBuLXAtMSkKb3V0bGllcnMgPC0gcmVkdWNlZF80LnJlc1thYnMocmVkdWNlZF80LnJlcyk+Y3JpdF0Kb3V0bGllcnMKYGBgCgoKCgpgYGB7cn0KIyMgb3V0bGllcnMgcmVtb3ZlZApvdXRsaWVyc19pbmRleCA8LSBhdHRyKG91dGxpZXJzLCAibmFtZXMiKQpvdXRsaWVyc19pbmRleCA8LSBhcy5udW1lcmljKG91dGxpZXJzX2luZGV4KQp0cmFpbl9ub19vdXRsaWVycyA8LSB0cmFpblstKG91dGxpZXJzX2luZGV4KSxdCgojbGV2ZXJhZ2VzIHJlbW92ZWQKbGVyZXJhZ2VzX2luZGV4IDwtIGF0dHIobGV2ZXJhZ2VzLCAibmFtZXMiKQpsZXJlcmFnZXNfaW5kZXggPC0gYXMubnVtZXJpYyhsZXJlcmFnZXNfaW5kZXgpCnRyYWluX25vX2xldmVyYWdlcyA8LSB0cmFpblstKGxlcmVyYWdlc19pbmRleCksXQoKIyBEREZGSVRTX2luZmx1ZW5jZQpEREZGSVRTX2luZGV4IDwtIGF0dHIoRERGRklUU19pbmZsdWVuY2UsICJuYW1lcyIpCkRERkZJVFNfaW5kZXggPC0gYXMubnVtZXJpYyhEREZGSVRTX2luZGV4KQp0cmFpbl9ub19EREZGSVRTIDwtIHRyYWluWy0oRERGRklUU19pbmRleCksXQoKIyBhbGwgIm5vbi1ub3JtYWwiIHJlbW92ZWQKYWxsX3NwZWNpYWwgPC0gYyhEREZGSVRTX2luZGV4LGxlcmVyYWdlc19pbmRleCxvdXRsaWVyc19pbmRleCkKdHJhaW5fbm90aGluZ19zcGVjaWFsIDwtIHRyYWluWy0oYWxsX3NwZWNpYWwpLF0KdHJhaW5fbm90aGluZ19zcGVjaWFsCgoKCgpgYGAKYGBge3J9CnZpZih0cmFpbltjKDIsMyw0LDcsOCw2LDksMTAsMTEpXSkKYGBgCgpgYGB7cn0KdHJhaW5fdGVtcDwtdHJhaW4KIyBhcy5mYWN0b3IodHJhaW5fdGVtcCRUeXBlKTwtbnVtZXJpYyh0cmFpbl90ZW1wJFR5cGUpCiN0cmFpbl90ZW1wCiMgYXMuZmFjdG9yCgp0cmFpbl90ZW1wJFR5cGUgPC0gYXMubnVtZXJpYyh0cmFpbl90ZW1wJFR5cGUpLTEKdHJhaW5fdGVtcCRUeXBlIDwtIGFzLmludGVnZXIodHJhaW5fdGVtcCRUeXBlKQp0cmFpbl90ZW1wCmBgYAoKCgojIyBjcmVhdGluZyByZWR1Y2VkIAoKYGBge3J9CnJlZHVjZWRfNF8zIDwtIGdsbShjYXRfcXVhbGl0eX52b2xhdGlsZS5hY2lkaXR5K2NpdHJpYy5hY2lkK3Jlc2lkdWFsLnN1Z2FyK3RvdGFsLnN1bGZ1ci5kaW94aWRlK2RlbnNpdHkrZnJlZS5zdWxmdXIuZGlveGlkZStwSCtzdWxwaGF0ZXMrYWxjb2hvbCtUeXBlLCBmYW1pbHk9Ymlub21pYWwsIGRhdGE9dHJhaW5fbm9fb3V0bGllcnMpCgpyZWR1Y2VkXzRfNF9sZXYgPC0gZ2xtKGNhdF9xdWFsaXR5fnZvbGF0aWxlLmFjaWRpdHkrY2l0cmljLmFjaWQrcmVzaWR1YWwuc3VnYXIrdG90YWwuc3VsZnVyLmRpb3hpZGUrZGVuc2l0eStmcmVlLnN1bGZ1ci5kaW94aWRlK3BIK3N1bHBoYXRlcythbGNvaG9sK1R5cGUsIGZhbWlseT1iaW5vbWlhbCwgZGF0YT10cmFpbl9ub19sZXZlcmFnZXMpCgpyZWR1Y2VkXzRfNV9EREZGSVRTIDwtIGdsbShjYXRfcXVhbGl0eX52b2xhdGlsZS5hY2lkaXR5K2NpdHJpYy5hY2lkK3Jlc2lkdWFsLnN1Z2FyK3RvdGFsLnN1bGZ1ci5kaW94aWRlK2RlbnNpdHkrZnJlZS5zdWxmdXIuZGlveGlkZStwSCtzdWxwaGF0ZXMrYWxjb2hvbCtUeXBlLCBmYW1pbHk9Ymlub21pYWwsIGRhdGE9dHJhaW5fbm9fRERGRklUUykKCnJlZHVjZWRfNF82X25vX3NwZWNpYWwgPC0gZ2xtKGNhdF9xdWFsaXR5fnZvbGF0aWxlLmFjaWRpdHkrY2l0cmljLmFjaWQrcmVzaWR1YWwuc3VnYXIrdG90YWwuc3VsZnVyLmRpb3hpZGUrZGVuc2l0eStmcmVlLnN1bGZ1ci5kaW94aWRlK3BIK3N1bHBoYXRlcythbGNvaG9sK1R5cGUsIGZhbWlseT1iaW5vbWlhbCwgZGF0YT10cmFpbl9ub3RoaW5nX3NwZWNpYWwpCiMgc3VtbWFyeShyZWR1Y2VkXzRfNl9ub19zcGVjaWFsKQoKCgpgYGAKCgpgYGB7cn0KIyMgY2hlY2tpbmcgY29saW5lYXJpdHkgLyBWSUYgc2NvcmVzCiMgcmVkdWNlZF80XzNfY29sIDwtIGRhdGEuZnJhbWUoJ3JlZHVjZWRfNF8zJyA9IGNoZWNrX2NvbGxpbmVhcml0eShyZWR1Y2VkXzRfMykpCiMgcmVkdWNlZF80XzNfY29sX1ZJRiA8LSByZWR1Y2VkXzRfM19jb2xbYygncmVkdWNlZF80XzMuVGVybScsJ3JlZHVjZWRfNF8zLlZJRicpXQojIHJlZHVjZWRfNF8zX2NvbF9WSUYKIyAKIyByZWR1Y2VkXzRfNF9sZXZfY29sIDwtIGRhdGEuZnJhbWUoJ3JlZHVjZWRfNF80X2xldicgPSBjaGVja19jb2xsaW5lYXJpdHkocmVkdWNlZF80XzRfbGV2KSkKIyByZWR1Y2VkXzRfNF9sZXZfY29sX1ZJRiA8LSByZWR1Y2VkXzRfNF9sZXZfY29sW2MoJ3JlZHVjZWRfNF80X2xldi5UZXJtJywncmVkdWNlZF80XzRfbGV2LlZJRicpXQojIHJlZHVjZWRfNF80X2xldl9jb2xfVklGCiMgCiMgCiMgcmVkdWNlZF80XzVfRERGRklUU19jb2wgPC0gZGF0YS5mcmFtZSgncmVkdWNlZF80XzVfRERGRklUUycgPSBjaGVja19jb2xsaW5lYXJpdHkocmVkdWNlZF80XzVfRERGRklUUykpCiMgcmVkdWNlZF80XzVfRERGRklUU19jb2xfVklGIDwtIHJlZHVjZWRfNF81X0RERkZJVFNfY29sW2MoJ3JlZHVjZWRfNF81X0RERkZJVFMuVGVybScsJ3JlZHVjZWRfNF81X0RERkZJVFMuVklGJyldCiMgcmVkdWNlZF80XzVfRERGRklUU19jb2xfVklGCiMgCiMgcmVkdWNlZF80XzZfbm9fc3BlY2lhbF9jb2wgPC0gZGF0YS5mcmFtZSgncmVkdWNlZF80XzZfbm9fc3BlY2lhbCcgPSBjaGVja19jb2xsaW5lYXJpdHkocmVkdWNlZF80XzZfbm9fc3BlY2lhbCkpCiMgcmVkdWNlZF80XzZfbm9fc3BlY2lhbF9jb2xfVklGIDwtIHJlZHVjZWRfNF82X25vX3NwZWNpYWxfY29sW2MoJ3JlZHVjZWRfNF82X25vX3NwZWNpYWwuVGVybScsJ3JlZHVjZWRfNF82X25vX3NwZWNpYWwuVklGJyldCiMgcmVkdWNlZF80XzZfbm9fc3BlY2lhbF9jb2xfVklGCiMgCiMgVklGX3N1bW1hcnkgPC0gZGF0YS5mcmFtZSgnMCc9cmVkdWNlZF80XzNfY29sX1ZJRlsncmVkdWNlZF80XzMuVGVybSddLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgJzEnPXJlZHVjZWRfNF8zX2NvbF9WSUZbJ3JlZHVjZWRfNF8zLlZJRiddLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgJzInPXJlZHVjZWRfNF80X2xldl9jb2xfVklGWydyZWR1Y2VkXzRfNF9sZXYuVklGJ10sCiMgICAgICAgICAgICAgICAgICAgICAgICAgICAnMyc9cmVkdWNlZF80XzVfRERGRklUU19jb2xfVklGWydyZWR1Y2VkXzRfNV9EREZGSVRTLlZJRiddLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgJzQnPXJlZHVjZWRfNF82X25vX3NwZWNpYWxfY29sX1ZJRlsncmVkdWNlZF80XzZfbm9fc3BlY2lhbC5WSUYnXSkKIyBjb2xuYW1lcyhWSUZfc3VtbWFyeSkgPC0gYygnUHJlZGljdG9yIFZhcmlhYmxlJywnNF8zLlZJRi5PdXRsaWVycycsJzRfNF9sZXYuVklGJywnNF81X0RERkZJVFMuVklGJywnNF82X25vX3NwZWNpYWwuVklGJykKIyBWSUZfc3VtbWFyeQoKIyMgVklGIGZvciBPdXRsaWVycwojIyMgY2F0X3F1YWxpdHl+dm9sYXRpbGUuYWNpZGl0eStjaXRyaWMuYWNpZCtyZXNpZHVhbC5zdWdhcit0b3RhbC5zdWxmdXIuZGlveGlkZStkZW5zaXR5K2ZyZWUuc3VsZnVyLmRpb3hpZGUrcEgrc3VscGhhdGVzK2FsY29ob2wrVHlwZQoKCiMKcmVnXzRfVklGX3Rlc3QgPC0gdmlmKHRyYWluX3RlbXBbYygxLDIsMyw0LDcsOCw2LDksMTAsMTEsMTIpXSkKcmVnXzRfMl9WSUZfdGVzdCA8LSB2aWYodHJhaW5fdGVtcFtjKDIsMyw0LDcsOCw2LDksMTAsMTEsMTIpXSkKb3V0bGllcnNfVklGIDwtIHZpZih0cmFpbl9ub19vdXRsaWVyc1tjKDIsMyw0LDcsOCw2LDksMTAsMTEsMTIpXSkKbGV2ZXJhZ2VfVklGIDwtIHZpZih0cmFpbl9ub19sZXZlcmFnZXNbYygyLDMsNCw3LDgsNiw5LDEwLDExLDEyKV0pCkRERkZJVFNfVklGIDwtIHZpZih0cmFpbl9ub19EREZGSVRTW2MoMiwzLDQsNyw4LDYsOSwxMCwxMSwxMildKQpub3RoaW5nX3NwZWNpYWwgPC0gdmlmKHRyYWluX25vdGhpbmdfc3BlY2lhbFtjKDIsMyw0LDcsOCw2LDksMTAsMTEsMTIpXSkKCgpyZWdfNF9WSUZfdGVzdAoKVklGX3N1bW1hcnlfdGVzdCA8LSBkYXRhLmZyYW1lKCdiZXN0X3Bvc3NpYmxlX1ZJRiAocG9zdCknPXJlZ180XzJfVklGX3Rlc3QsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnb3V0bGllcnNfVklGJz1vdXRsaWVyc19WSUYsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbGV2ZXJhZ2VfVklGJz1sZXZlcmFnZV9WSUYsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnRERGRklUU19WSUYnPSBEREZGSVRTX1ZJRiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdub3RoaW5nX3NwZWNpYWwnPW5vdGhpbmdfc3BlY2lhbCkKVklGX3N1bW1hcnlfdGVzdAoKYGBgCgoKYGBge3J9CiMjZXZhbHVhdGluZyBtb2RlbApSZWR1Y2VkNF8zX0FJQ190cmFpbiA8LSByZWR1Y2VkXzRfMyRhaWMKCiMjcHJlZGljdGVkIHF1YWxpdHkgZm9yIHRlc3QgZGF0YSBiYXNlZCBvbiB0cmFpbmluZyBkYXRhCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF8zLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQoKcmVkdWNlZF80XzNfZXJyb3IgPC0gdGFibGUodGVzdCRjYXRfcXVhbGl0eSwgcHJlZHM+MC42KQoKZXZ1bGF0aW9uX3N1bW1hcnlfNF8zIDwtIGRhdGEuZnJhbWUoCiAgYXR0ZW1wdCA9ICdyZWR1Y2VkXzRfM19lcnJvcl9vdXRsaWVycycsCiAgQUlDID0gUmVkdWNlZDRfM19BSUNfdHJhaW4sCiAgUFJFU1MgPSBnZXRfcHJlc3MocmVkdWNlZF80XzMpLAogICdGYWxzZSBwb3NpdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfM19lcnJvclszXS8ocmVkdWNlZF80XzNfZXJyb3JbMV0rcmVkdWNlZF80XzNfZXJyb3JbM10pLDMpLAogICdGYWxzZSBuZWdhdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfM19lcnJvclsyXS8ocmVkdWNlZF80XzNfZXJyb3JbMl0rcmVkdWNlZF80XzNfZXJyb3JbNF0pLDMpLAogICdFcnJvciBSYXRlJyA9IHJvdW5kKChyZWR1Y2VkXzRfM19lcnJvclsyXStyZWR1Y2VkXzRfM19lcnJvclszXSkvKHJlZHVjZWRfNF8zX2Vycm9yWzFdK3JlZHVjZWRfNF8zX2Vycm9yWzJdK3JlZHVjZWRfNF8zX2Vycm9yWzNdK3JlZHVjZWRfNF8zX2Vycm9yWzRdKSwzKQopCgpldnVsYXRpb25fc3VtbWFyeSA8LSByYmluZChldnVsYXRpb25fc3VtbWFyeSxldnVsYXRpb25fc3VtbWFyeV80XzMpCmV2dWxhdGlvbl9zdW1tYXJ5CmBgYAoKCmBgYHtyfQojI2V2YWx1YXRpbmcgbW9kZWwgbGV2ZXJhZ2UKcmVkdWNlZF80XzRfbGV2X0FJQ190cmFpbiA8LSByZWR1Y2VkXzRfNF9sZXYkYWljCgojI3ByZWRpY3RlZCBxdWFsaXR5IGZvciB0ZXN0IGRhdGEgYmFzZWQgb24gdHJhaW5pbmcgZGF0YQpwcmVkczwtcHJlZGljdChyZWR1Y2VkXzRfNF9sZXYsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCgpyZWR1Y2VkXzRfNF9sZXZfZXJyb3IgPC0gdGFibGUodGVzdCRjYXRfcXVhbGl0eSwgcHJlZHM+MC42NSkKCmV2dWxhdGlvbl9zdW1tYXJ5XzRfNF9sZXYgPC0gZGF0YS5mcmFtZSgKICBhdHRlbXB0ID0gJ3JlZHVjZWRfNF80X2xldl9lcnJvcicsCiAgQUlDID0gcmVkdWNlZF80XzRfbGV2X0FJQ190cmFpbiwKICBQUkVTUyA9IGdldF9wcmVzcyhyZWR1Y2VkXzRfNF9sZXYpLAogICdGYWxzZSBwb3NpdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbM10vKHJlZHVjZWRfNF80X2xldl9lcnJvclsxXStyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbM10pLDMpLAogICdGYWxzZSBuZWdhdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbMl0vKHJlZHVjZWRfNF80X2xldl9lcnJvclsyXStyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbNF0pLDMpLAogICdFcnJvciBSYXRlJyA9IHJvdW5kKChyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbMl0rcmVkdWNlZF80XzRfbGV2X2Vycm9yWzNdKS8ocmVkdWNlZF80XzRfbGV2X2Vycm9yWzFdK3JlZHVjZWRfNF80X2xldl9lcnJvclsyXStyZWR1Y2VkXzRfNF9sZXZfZXJyb3JbM10rcmVkdWNlZF80XzRfbGV2X2Vycm9yWzRdKSwzKQopCgpldnVsYXRpb25fc3VtbWFyeSA8LSByYmluZChldnVsYXRpb25fc3VtbWFyeSxldnVsYXRpb25fc3VtbWFyeV80XzRfbGV2KQpldnVsYXRpb25fc3VtbWFyeQpgYGAKCgoKYGBge3J9CiMjZXZhbHVhdGluZyBtb2RlbCBEREZGSVRTCnJlZHVjZWRfNF81X0RERkZJVFNfQUlDX3RyYWluIDwtIHJlZHVjZWRfNF81X0RERkZJVFMkYWljCgojI3ByZWRpY3RlZCBxdWFsaXR5IGZvciB0ZXN0IGRhdGEgYmFzZWQgb24gdHJhaW5pbmcgZGF0YQpwcmVkczwtcHJlZGljdChyZWR1Y2VkXzRfNV9EREZGSVRTLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQoKcmVkdWNlZF80XzVfRERGRklUU19lcnJvciA8LSB0YWJsZSh0ZXN0JGNhdF9xdWFsaXR5LCBwcmVkcz4wLjcpCgpldnVsYXRpb25fc3VtbWFyeV80XzVfRERGRklUUyA8LSBkYXRhLmZyYW1lKAogIGF0dGVtcHQgPSAncmVkdWNlZF80XzVfRERGRklUU19lcnJvcicsCiAgQUlDID0gcmVkdWNlZF80XzVfRERGRklUU19BSUNfdHJhaW4sCiAgUFJFU1MgPSBnZXRfcHJlc3MocmVkdWNlZF80XzVfRERGRklUUyksCiAgJ0ZhbHNlIHBvc2l0aXZlJyA9IHJvdW5kKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbM10vKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbMV0rcmVkdWNlZF80XzVfRERGRklUU19lcnJvclszXSksMyksCiAgJ0ZhbHNlIG5lZ2F0aXZlJyA9IHJvdW5kKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbMl0vKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbMl0rcmVkdWNlZF80XzVfRERGRklUU19lcnJvcls0XSksMyksCiAgJ0Vycm9yIFJhdGUnID0gcm91bmQoKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbMl0rcmVkdWNlZF80XzVfRERGRklUU19lcnJvclszXSkvKHJlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbMV0rcmVkdWNlZF80XzVfRERGRklUU19lcnJvclsyXStyZWR1Y2VkXzRfNV9EREZGSVRTX2Vycm9yWzNdK3JlZHVjZWRfNF81X0RERkZJVFNfZXJyb3JbNF0pLDMpCikKCmV2dWxhdGlvbl9zdW1tYXJ5IDwtIHJiaW5kKGV2dWxhdGlvbl9zdW1tYXJ5LGV2dWxhdGlvbl9zdW1tYXJ5XzRfNV9EREZGSVRTKQpldnVsYXRpb25fc3VtbWFyeQpgYGAKCmBgYHtyfQojI2V2YWx1YXRpbmcgbW9kZWwgRERGRklUUwpyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX0FJQ190cmFpbiA8LSByZWR1Y2VkXzRfNl9ub19zcGVjaWFsJGFpYwoKIyNwcmVkaWN0ZWQgcXVhbGl0eSBmb3IgdGVzdCBkYXRhIGJhc2VkIG9uIHRyYWluaW5nIGRhdGEKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzZfbm9fc3BlY2lhbCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKCnJlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3IgPC0gdGFibGUodGVzdCRjYXRfcXVhbGl0eSwgcHJlZHM+MC44KQoKZXZ1bGF0aW9uX3N1bW1hcnlfNF82X25vX3NwZWNpYWwgPC0gZGF0YS5mcmFtZSgKICBhdHRlbXB0ID0gJ3JlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3InLAogIEFJQyA9IHJlZHVjZWRfNF82X25vX3NwZWNpYWxfQUlDX3RyYWluLAogIFBSRVNTID0gZ2V0X3ByZXNzKHJlZHVjZWRfNF82X25vX3NwZWNpYWwpLAogICdGYWxzZSBwb3NpdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzNdLyhyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzFdK3JlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3JbM10pLDMpLAogICdGYWxzZSBuZWdhdGl2ZScgPSByb3VuZChyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzJdLyhyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzJdK3JlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3JbNF0pLDMpLAogICdFcnJvciBSYXRlJyA9IHJvdW5kKChyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzJdK3JlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3JbM10pLyhyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzFdK3JlZHVjZWRfNF82X25vX3NwZWNpYWxfZXJyb3JbMl0rcmVkdWNlZF80XzZfbm9fc3BlY2lhbF9lcnJvclszXStyZWR1Y2VkXzRfNl9ub19zcGVjaWFsX2Vycm9yWzRdKSwzKQopCgpldnVsYXRpb25fc3VtbWFyeSA8LSByYmluZChldnVsYXRpb25fc3VtbWFyeSxldnVsYXRpb25fc3VtbWFyeV80XzZfbm9fc3BlY2lhbCkKZXZ1bGF0aW9uX3N1bW1hcnkKYGBgCgoKIyMgUk9DIEN1cnZlcyBhbmQgQVVDCmBgYHtyfQojIyByZWR1Y2VkXzEKIyBkZXRhY2gocGFja2FnZTpwZXJmb3JtYW5jZSwgdW5sb2FkPVRSVUUpCiMjIEZZSSB0aGUgcGVyZm9ybWFuY2UgcGFja2FnZSBjYXVzZXMgUk9DIGN1cnZlcyB0byBub3Qgd29yawpsaWJyYXJ5KFJPQ1IpCgoKCiMgcmVkdWNlZF8xCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfMSxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfMSIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzwtcGVyZm9ybWFuY2UocmF0ZXMsIG1lYXN1cmUgPSAiYXVjIikKcmVkdWNlZF8xX2F1YyA8LSBhdWNAeS52YWx1ZXMKCiMjIHJlZHVjZWRfNApwcmVkczwtcHJlZGljdChyZWR1Y2VkXzQsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCnJhdGVzNDwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXM0LG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNCIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzQ8LXBlcmZvcm1hbmNlKHJhdGVzNCwgbWVhc3VyZSA9ICJhdWMiKQpyZWR1Y2VkXzRfYXVjIDwtIGF1YzRAeS52YWx1ZXMKCiMjIHJlZHVjZWRfNF8yCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF8yLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQpwbG90KHJvY19yZXN1bHQsIG1haW49IlJPQyBDdXJ2ZSBmb3IgcmVkdWNlZF80XzIiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM0XzI8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF8yX2F1YyA8LSBhdWM0XzJAeS52YWx1ZXMKCiMjIHJlZHVjZWRfNF8zCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF8zLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQpwbG90KHJvY19yZXN1bHQsIG1haW49IlJPQyBDdXJ2ZSBmb3IgcmVkdWNlZF80XzMiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF8zX2F1YyA8LSBhdWNAeS52YWx1ZXMKCiMjIHJlZHVjZWRfNF80X2xldiAKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzRfbGV2LG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQpwbG90KHJvY19yZXN1bHQsIG1haW49IlJPQyBDdXJ2ZSBmb3IgcmVkdWNlZF80XzRfbGV2IikKbGluZXMoeCA9IGMoMCwxKSwgeSA9IGMoMCwxKSwgY29sPSJyZWQiKQoKYXVjPC1wZXJmb3JtYW5jZShyYXRlcywgbWVhc3VyZSA9ICJhdWMiKQpyZWR1Y2VkXzRfNF9sZXZfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKIyMgcmVkdWNlZF80XzVfRERGRklUUyAKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzVfRERGRklUUyxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNF81X0RERkZJVFMiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF81X0RERkZJVFNfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKIyMgcmVkdWNlZF80XzZfbm9fc3BlY2lhbCAKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzZfbm9fc3BlY2lhbCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNF82X25vX3NwZWNpYWwiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF82X25vX3NwZWNpYWxfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKQVVDX3N1bW1hcnkgPC0gZGF0YS5mcmFtZSgncmVkdWNlZF8xJz1yZWR1Y2VkXzFfYXVjLAogICAgICAgICAgICAgICAgICAgICAgICAgICdyZWR1Y2VkXzQnPXJlZHVjZWRfNF9hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF8yJz1yZWR1Y2VkXzRfMl9hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF8zJz1yZWR1Y2VkXzRfM19hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF80X2xldic9cmVkdWNlZF80XzRfbGV2X2F1YywKICAgICAgICAgICAgICAgICAgICAgICAgICAncmVkdWNlZF80XzVfRERGRklUUyc9cmVkdWNlZF80XzVfRERGRklUU19hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF82X25vX3NwZWNpYWwnPXJlZHVjZWRfNF82X25vX3NwZWNpYWxfYXVjKQpjb2xuYW1lcyhBVUNfc3VtbWFyeSkgPC0gYygncmVkdWNlZF8xJywncmVkdWNlZF80JywncmVkdWNlZF80XzInLCdyZWR1Y2VkXzRfMycsJ3JlZHVjZWRfNF80X2xldicsJ3JlZHVjZWRfNF81X0RERkZJVFMnLCdyZWR1Y2VkXzRfNl9ub19zcGVjaWFsJykKCkFVQ19zdW1tYXJ5CmBgYAoKCgoKCiMjIFJ5YW4ncyBwYXJ0IHN0YXJ0cyBoZXJlCgojIyMgVGhlIGdvYWwgaXMgdG8gbWFrZSAzIG1vZGVsczogT25lIGZvciBqdXN0IHdoaXRlLCBvbmUgZm9yIGp1c3QgcmVkLCBhbmQgb25lIHdpdGggaW50ZXJhY3Rpb24gdGVybXMgd2l0aCB0aGUgdHlwZSBvZiB3aW5lLiAKCiMjIyMgQWZ0ZXIgdGhhdCwgdGhlIG1vZGVscyB3aWxsIGJlIHRyYWluZWQgb24gdGhlIGZpbHRlcmVkIGRhdGFzZXRzIGFuZCB0aGUgcmVzdWx0aW5nIHNjb3JlcyB3aWxsIGJlIGFkZGVkIHRvIHRoZSBldmFsdWF0aW9uIHN1bW1hcnkuCgoKIyMgUmVkIHdpbmUgb25seSBtb2RlbApgYGB7cn0KcmVnZnVsbF9SZWQ8LWdsbShjYXRfcXVhbGl0eX4uLCBmYW1pbHk9ImJpbm9taWFsIiwgZGF0YT10cmFpbl9SZWRfTm9UeXBlKQpyZWdudWxsX1JlZDwtZ2xtKGNhdF9xdWFsaXR5fjEsIGZhbWlseT0iYmlub21pYWwiLCBkYXRhPXRyYWluX1JlZF9Ob1R5cGUpCnN0ZXAocmVnbnVsbF9SZWQsIHNjb3BlPWxpc3QobG93ZXI9cmVnbnVsbF9SZWQsIHVwcGVyPXJlZ2Z1bGxfUmVkKSwgZGlyZWN0aW9uPSJmb3J3YXJkIikKCmBgYAoKClRoZSBtb2RlbCBsb29rcyBncmVhdCBhZnRlciB0aGUgZm93YXJkIHNlbGVjdGlvbiEgVGltZSB0byB0ZXN0IGFuZCBhZGQgdG8gdGhlIGV2YWx1YXRpb24gc3VtbWFyeS4KYGBge3J9Cm1vZGVsMV9SZWQ8LWdsbShmb3JtdWxhID0gY2F0X3F1YWxpdHkgfiBhbGNvaG9sICsgdm9sYXRpbGUuYWNpZGl0eSArIHRvdGFsLnN1bGZ1ci5kaW94aWRlICsgCiAgICBzdWxwaGF0ZXMgKyBmcmVlLnN1bGZ1ci5kaW94aWRlICsgY2hsb3JpZGVzLCBmYW1pbHkgPSAiYmlub21pYWwiLCAKICAgIGRhdGEgPSB0cmFpbl9SZWRfTm9UeXBlKQoKc3VtbWFyeShtb2RlbDFfUmVkKQoKCmBgYAoKYGBge3J9CgojI2V2YWx1YXRpbmcgbW9kZWwKbW9kZWwxX1JlZF9BSUNfdHJhaW4gPC0gbW9kZWwxX1JlZCRhaWMKIyNwcmVkaWN0ZWQgcXVhbGl0eSBmb3IgdGVzdCBkYXRhIGJhc2VkIG9uIHRyYWluaW5nIGRhdGEKdGVzdF9SZWRfTm9UeXBlPC1zdWJzZXQodGVzdCwgVHlwZSA9PSAwLCBzZWxlY3Q9LWMoVHlwZSkpCnByZWRzPC1wcmVkaWN0KG1vZGVsMV9SZWQsbmV3ZGF0YT10ZXN0X1JlZF9Ob1R5cGUsIHR5cGU9InJlc3BvbnNlIikKbW9kZWwxX1JlZF9lcnJvciA8LSB0YWJsZSh0ZXN0X1JlZF9Ob1R5cGUkY2F0X3F1YWxpdHksIHByZWRzPjAuNykKI0N1cnZlcwpldnVsYXRpb25fc3VtbWFyeV8xUiA8LSBkYXRhLmZyYW1lKAogIGF0dGVtcHQgPSAnbW9kZWwxX1JlZCcsCiAgQUlDID0gbW9kZWwxX1JlZF9BSUNfdHJhaW4sCiAgUFJFU1MgPSBnZXRfcHJlc3MobW9kZWwxX1JlZCksCiAgJ0ZhbHNlIHBvc2l0aXZlJyA9IHJvdW5kKG1vZGVsMV9SZWRfZXJyb3JbM10vKG1vZGVsMV9SZWRfZXJyb3JbMV0rbW9kZWwxX1JlZF9lcnJvclszXSksMyksCiAgJ0ZhbHNlIG5lZ2F0aXZlJyA9IHJvdW5kKG1vZGVsMV9SZWRfZXJyb3JbMl0vKG1vZGVsMV9SZWRfZXJyb3JbMl0rbW9kZWwxX1JlZF9lcnJvcls0XSksMyksCiAgJ0Vycm9yIFJhdGUnID0gcm91bmQoKG1vZGVsMV9SZWRfZXJyb3JbMl0rbW9kZWwxX1JlZF9lcnJvclszXSkvKG1vZGVsMV9SZWRfZXJyb3JbMV0rbW9kZWwxX1JlZF9lcnJvclsyXSttb2RlbDFfUmVkX2Vycm9yWzNdK21vZGVsMV9SZWRfZXJyb3JbNF0pLDMpCikKCmNvbXBhcmVfbW9kZWxzPC1yYmluZChldnVsYXRpb25fc3VtbWFyeVsxLF0sZXZ1bGF0aW9uX3N1bW1hcnlfMVIpCmNvbXBhcmVfbW9kZWxzCgpldnVsYXRpb25fc3VtbWFyeSA8LSByYmluZChldnVsYXRpb25fc3VtbWFyeSxldnVsYXRpb25fc3VtbWFyeV8xUikKZXZ1bGF0aW9uX3N1bW1hcnkKCmBgYApgYGB7cn0KIyBtb2RlbDFfUmVkCmxpYnJhcnkoUk9DUikKcHJlZHM8LXByZWRpY3QobW9kZWwxX1JlZCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9SZWQiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCm1vZGVsMV9SZWRfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKYGBgCgojIyBXaGl0ZSB3aW5lIG9ubHkgbW9kZWwKCmBgYHtyfQpyZWdmdWxsX1doaXRlPC1nbG0oY2F0X3F1YWxpdHl+LiwgZmFtaWx5PSJiaW5vbWlhbCIsIGRhdGE9dHJhaW5fV2hpdGVfTm9UeXBlKQpyZWdudWxsX1doaXRlPC1nbG0oY2F0X3F1YWxpdHl+MSxmYW1pbHk9ImJpbm9taWFsIiwgZGF0YT10cmFpbl9XaGl0ZV9Ob1R5cGUpCnN0ZXAocmVnbnVsbF9XaGl0ZSwgc2NvcGU9bGlzdChsb3dlcj1yZWdudWxsX1doaXRlLCB1cHBlcj1yZWdmdWxsX1doaXRlKSwgZGlyZWN0aW9uPSJmb3J3YXJkIikKYGBgCgoKVGhlIG1vZGVsIGxvb2tzIGdvb2QgYWZ0ZXIgdGhlIGZvd2FyZCBzZWxlY3Rpb24sIGJ1dCB0aGUgcHJlZGljdG9yIGZpeGVkLmFjaWRpdHkgY2FuIGJlIHJlbW92ZWQuIFRoZSBkZW5zaXR5IFZJRiBpcyBhYm92ZSB0ZW4sIGJ1dCBqc3V0IGJhcmVseS4gRm9yIG5vdywgaXQgd2lsbCBiZSBsZWZ0IGluLiBUaW1lIHRvIHRlc3QgYW5kIGFkZCB0byB0aGUgZXZhbHVhdGlvbiBzdW1tYXJ5LgpgYGB7cn0KbW9kZWwxX1doaXRlPC1nbG0oZm9ybXVsYSA9IGNhdF9xdWFsaXR5IH4gYWxjb2hvbCArIHZvbGF0aWxlLmFjaWRpdHkgKyByZXNpZHVhbC5zdWdhciArIAogICAgZml4ZWQuYWNpZGl0eSArIHN1bHBoYXRlcyArIGZyZWUuc3VsZnVyLmRpb3hpZGUgKyBkZW5zaXR5ICsgCiAgICBwSCwgZmFtaWx5ID0gImJpbm9taWFsIiwgZGF0YSA9IHRyYWluX1doaXRlX05vVHlwZSkKCgpzdW1tYXJ5KG1vZGVsMV9XaGl0ZSkKCgptb2RlbDFfV2hpdGU8LWdsbShmb3JtdWxhID0gY2F0X3F1YWxpdHkgfiBhbGNvaG9sICsgdm9sYXRpbGUuYWNpZGl0eSArIHJlc2lkdWFsLnN1Z2FyICsgc3VscGhhdGVzICsgZnJlZS5zdWxmdXIuZGlveGlkZSArIGRlbnNpdHkgKyAKICAgIHBILCBmYW1pbHkgPSAiYmlub21pYWwiLCBkYXRhID0gdHJhaW5fV2hpdGVfTm9UeXBlKQoKc3VtbWFyeShtb2RlbDFfV2hpdGUpCgoKYGBgCgoKYGBge3J9CgojI2V2YWx1YXRpbmcgbW9kZWwKbW9kZWwxX1doaXRlX0FJQ190cmFpbiA8LSBtb2RlbDFfV2hpdGUkYWljCiMjcHJlZGljdGVkIHF1YWxpdHkgZm9yIHRlc3QgZGF0YSBiYXNlZCBvbiB0cmFpbmluZyBkYXRhCnRlc3RfV2hpdGVfTm9UeXBlPC1zdWJzZXQodGVzdCwgVHlwZSA9PSAxLCBzZWxlY3Q9LWMoVHlwZSkpCnByZWRzPC1wcmVkaWN0KG1vZGVsMV9XaGl0ZSxuZXdkYXRhPXRlc3RfV2hpdGVfTm9UeXBlLCB0eXBlPSJyZXNwb25zZSIpCm1vZGVsMV9XaGl0ZV9lcnJvciA8LSB0YWJsZSh0ZXN0X1doaXRlX05vVHlwZSRjYXRfcXVhbGl0eSwgcHJlZHM+MC43KQojQ3VydmVzCmV2dWxhdGlvbl9zdW1tYXJ5XzFXIDwtIGRhdGEuZnJhbWUoCiAgYXR0ZW1wdCA9ICdtb2RlbDFfV2hpdGUnLAogIEFJQyA9IG1vZGVsMV9XaGl0ZV9BSUNfdHJhaW4sCiAgUFJFU1MgPSBnZXRfcHJlc3MobW9kZWwxX1doaXRlKSwKICAnRmFsc2UgcG9zaXRpdmUnID0gcm91bmQobW9kZWwxX1doaXRlX2Vycm9yWzNdLyhtb2RlbDFfV2hpdGVfZXJyb3JbMV0rbW9kZWwxX1doaXRlX2Vycm9yWzNdKSwzKSwKICAnRmFsc2UgbmVnYXRpdmUnID0gcm91bmQobW9kZWwxX1doaXRlX2Vycm9yWzJdLyhtb2RlbDFfV2hpdGVfZXJyb3JbMl0rbW9kZWwxX1doaXRlX2Vycm9yWzRdKSwzKSwKICAnRXJyb3IgUmF0ZScgPSByb3VuZCgobW9kZWwxX1doaXRlX2Vycm9yWzJdK21vZGVsMV9XaGl0ZV9lcnJvclszXSkvKG1vZGVsMV9XaGl0ZV9lcnJvclsxXSttb2RlbDFfV2hpdGVfZXJyb3JbMl0rbW9kZWwxX1doaXRlX2Vycm9yWzNdK21vZGVsMV9XaGl0ZV9lcnJvcls0XSksMykKKQoKZXZ1bGF0aW9uX3N1bW1hcnkgPC0gcmJpbmQoZXZ1bGF0aW9uX3N1bW1hcnksZXZ1bGF0aW9uX3N1bW1hcnlfMVcpCmV2dWxhdGlvbl9zdW1tYXJ5CgoKY29tcGFyZV9tb2RlbHM8LXJiaW5kKGNvbXBhcmVfbW9kZWxzLGV2dWxhdGlvbl9zdW1tYXJ5XzFXKQpjb21wYXJlX21vZGVscwoKYGBgCgoKCiMjIFRoZSBtb2RlbCB3aXRoIGludGVyYWN0aW9uIHRlcm1zCgpgYGB7cn0KcmVnZnVsbF9pbnQ8LWdsbShjYXRfcXVhbGl0eX4uKlR5cGUsIGZhbWlseT0iYmlub21pYWwiLCBkYXRhPXRyYWluKQpyZWdudWxsX2ludDwtZ2xtKGNhdF9xdWFsaXR5fjEsZmFtaWx5PSJiaW5vbWlhbCIsIGRhdGE9dHJhaW4pCnN0ZXAocmVnbnVsbF9pbnQsIHNjb3BlPWxpc3QobG93ZXI9cmVnbnVsbF9pbnQsIHVwcGVyPXJlZ2Z1bGxfaW50KSwgZGlyZWN0aW9uPSJmb3J3YXJkIikKYGBgCgoKVGhlIGZvcndhcmQgc3RlcCBwcm9jZXNzIGRyb3BwZWQgKyBzdWxwaGF0ZXM6VHlwZSwgZml4ZWQuYWNpZGl0eSwgYWxjb2hvbDpUeXBlLCBhbmQgY2l0cmljLmFjaWQgIEJ5IHRoZSBoaWVyYXJjaGljYWwgcHJpbmNpcGxlLCB0aGUgdHdvIG5vbi1pbnRlcmFjdGl2ZSB0ZXJtcyBuZWVkIHRvIGJlIGFkZGVkIGJhY2sgYmVjYXVzZSB0aGVpciBoYXZlIGludGVyYWN0aW9uIHRlcm1zIGFyZSBpbiB0aGUgbW9kZWwuCmBgYHtyfQogbW9kZWwxX2ludDwtZ2xtKGZvcm11bGEgPSBjYXRfcXVhbGl0eSB+IGFsY29ob2wgKyB2b2xhdGlsZS5hY2lkaXR5ICsgZGVuc2l0eSArIAogICAgc3VscGhhdGVzICsgcmVzaWR1YWwuc3VnYXIgKyAgVHlwZSArIHRvdGFsLnN1bGZ1ci5kaW94aWRlICsgCiAgICBmcmVlLnN1bGZ1ci5kaW94aWRlICsgcEggKyBjaGxvcmlkZXMgKyB2b2xhdGlsZS5hY2lkaXR5OlR5cGUgKyAKICAgIFR5cGU6dG90YWwuc3VsZnVyLmRpb3hpZGUgKyBkZW5zaXR5OlR5cGUgKyByZXNpZHVhbC5zdWdhcjpUeXBlICsgCiAgICBUeXBlOnBIICsgVHlwZTpjaGxvcmlkZXMgKyBUeXBlOmZyZWUuc3VsZnVyLmRpb3hpZGUsIGZhbWlseSA9ICJiaW5vbWlhbCIsIAogICAgZGF0YSA9IHRyYWluKQpzdW1tYXJ5KG1vZGVsMV9pbnQpCgpgYGAKCgpgYGB7cn0KCiMjZXZhbHVhdGluZyBtb2RlbAptb2RlbDFfaW50X0FJQ190cmFpbiA8LSBtb2RlbDFfaW50JGFpYwojI3ByZWRpY3RlZCBxdWFsaXR5IGZvciB0ZXN0IGRhdGEgYmFzZWQgb24gdHJhaW5pbmcgZGF0YQpwcmVkczwtcHJlZGljdChtb2RlbDFfaW50LG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQptb2RlbDFfaW50X2Vycm9yIDwtIHRhYmxlKHRlc3QkY2F0X3F1YWxpdHksIHByZWRzPjAuNykKI0N1cnZlcwpldnVsYXRpb25fc3VtbWFyeV8xaW50IDwtIGRhdGEuZnJhbWUoCiAgYXR0ZW1wdCA9ICdtb2RlbDFfaW50JywKICBBSUMgPSBtb2RlbDFfaW50X0FJQ190cmFpbiwKICBQUkVTUyA9IGdldF9wcmVzcyhtb2RlbDFfaW50KSwKICAnRmFsc2UgcG9zaXRpdmUnID0gcm91bmQobW9kZWwxX2ludF9lcnJvclszXS8obW9kZWwxX2ludF9lcnJvclsxXSttb2RlbDFfaW50X2Vycm9yWzNdKSwzKSwKICAnRmFsc2UgbmVnYXRpdmUnID0gcm91bmQobW9kZWwxX2ludF9lcnJvclsyXS8obW9kZWwxX2ludF9lcnJvclsyXSttb2RlbDFfaW50X2Vycm9yWzRdKSwzKSwKICAnRXJyb3IgUmF0ZScgPSByb3VuZCgobW9kZWwxX2ludF9lcnJvclsyXSttb2RlbDFfaW50X2Vycm9yWzNdKS8obW9kZWwxX2ludF9lcnJvclsxXSttb2RlbDFfaW50X2Vycm9yWzJdK21vZGVsMV9pbnRfZXJyb3JbM10rbW9kZWwxX2ludF9lcnJvcls0XSksMykKKQoKZXZ1bGF0aW9uX3N1bW1hcnkgPC0gcmJpbmQoZXZ1bGF0aW9uX3N1bW1hcnksZXZ1bGF0aW9uX3N1bW1hcnlfMWludCkKZXZ1bGF0aW9uX3N1bW1hcnkKCmNvbXBhcmVfbW9kZWxzPC1yYmluZChjb21wYXJlX21vZGVscyxldnVsYXRpb25fc3VtbWFyeV8xaW50KQpjb21wYXJlX21vZGVscwoKYGBgCgpgYGB7cn0KY29tcGFyZV9tb2RlbHM8LWNvbXBhcmVfbW9kZWxzJT4lIAogIHJlbmFtZSgKICAgIE1vZGVsID0gYXR0ZW1wdAogICAgKQoKYGBgCgoKYGBge3J9CmxpYnJhcnkoZGF0YS50YWJsZSkKbGlicmFyeShkcGx5cikKbGlicmFyeShmb3JtYXR0YWJsZSkKbGlicmFyeSh0aWR5cikKY3VzdG9tR3JlZW4wID0gIiNEZUY3RTkiCgpjdXN0b21HcmVlbiA9ICIjNzFDQTk3IgoKY3VzdG9tUmVkID0gIiNmZjdmN2YiCgpgYGAKCgoKCgoKCgoKCgoKCkNyZWF0aW5nIHRoZSBST0MgY3VydmVzIGFuZCBBVUMgZm9yIHRoZSAzIG5ldyBtb2RlbHMuCmBgYHtyfQojIG1vZGVsMV9SZWQKcHJlZHM8LXByZWRpY3QobW9kZWwxX1JlZCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9SZWQiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCm1vZGVsMV9SZWRfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKCiMgbW9kZWwxX1doaXRlCnByZWRzPC1wcmVkaWN0KG1vZGVsMV9XaGl0ZSxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9XaGl0ZSIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzwtcGVyZm9ybWFuY2UocmF0ZXMsIG1lYXN1cmUgPSAiYXVjIikKbW9kZWwxX1doaXRlX2F1YyA8LSBhdWNAeS52YWx1ZXMKCgojIG1vZGVsMV9pbnQKcHJlZHM8LXByZWRpY3QobW9kZWwxX2ludCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9pbnQiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCm1vZGVsMV9pbnRfYXVjIDwtIGF1Y0B5LnZhbHVlcwpgYGAKCgpgYGB7cn0KcmVkdWNlZF8xX2F1Ywptb2RlbDFfUmVkX2F1Ywptb2RlbDFfV2hpdGVfYXVjCm1vZGVsMV9pbnRfYXVjCmBgYAoKCgoKCgoKCgoKCgoKCgoKCgoKCgojIyBUaGlzIGlzIHRoZSBvbmUgbGluZXJzIHRoYXQgcnVuIHRoZSB0YWJsZXMgYW5kIGZpZ3VyZXMhCgoKCgoKYGBge3J9CgojI2NyZWF0ZSBoZWF0IG1hcCBDb25zb2xpZGF0ZWQKZ2dwbG90KGRhdGEgPSBtZWx0ZWRfY29yX3RyYWluLCBhZXMoeD1WYXIxLCB5PVZhcjIsIGZpbGw9dmFsdWUpKSArIAogIGdlb21fdGlsZShjb2xvciA9ICJ3aGl0ZSIpKwogIHNjYWxlX2ZpbGxfZ3JhZGllbnQyKGxvdyA9ICJibHVlIiwgaGlnaCA9ICJyZWQiLCBtaWQgPSAid2hpdGUiLCBtaWRwb2ludCA9IDAsIGxpbWl0ID0gYygtMSwxKSwgc3BhY2UgPSAiTGFiIiwgbmFtZT0iUGVhcnNvblxuQ29ycmVsYXRpb24iKSArCiAgdGhlbWVfbWluaW1hbCgpKyAKICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCB2anVzdCA9IDEsIHNpemUgPSAxMiwgaGp1c3QgPSAxKSkrCiAgY29vcmRfZml4ZWQoKSsKICBsYWJzKHRpdGxlID0gJ0NvbnNvbGlkYXRlZCAoQm90aCBSZWQgYW5kIFdoaXRlKScpCgpgYGAKCmBgYHtyfQojI2NyZWF0ZSBoZWF0IG1hcCBXaGl0ZQpnZ3Bsb3QoZGF0YSA9IG1lbHRlZF9jb3JfdHJhaW5fd2hpdGUsIGFlcyh4PVZhcjEsIHk9VmFyMiwgZmlsbD12YWx1ZSkpICsKICBnZW9tX3RpbGUoY29sb3IgPSAid2hpdGUiKSsKICBzY2FsZV9maWxsX2dyYWRpZW50Mihsb3cgPSAiYmx1ZSIsIGhpZ2ggPSAicmVkIiwgbWlkID0gIndoaXRlIiwgbWlkcG9pbnQgPSAwLCBsaW1pdCA9IGMoLTEsMSksIHNwYWNlID0gIkxhYiIsIG5hbWU9IlBlYXJzb25cbkNvcnJlbGF0aW9uIikgKwogIHRoZW1lX21pbmltYWwoKSsKICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCB2anVzdCA9IDEsIHNpemUgPSAxMiwgaGp1c3QgPSAxKSkrCiAgY29vcmRfZml4ZWQoKSsKICBsYWJzKHRpdGxlID0gJ1doaXRlIFdpbmUnKQoKYGBgCgpgYGB7cn0KIyNjcmVhdGUgaGVhdCBtYXAgUmVkCmdncGxvdChkYXRhID0gbWVsdGVkX2Nvcl90cmFpbl9SZWQsIGFlcyh4PVZhcjEsIHk9VmFyMiwgZmlsbD12YWx1ZSkpICsKICBnZW9tX3RpbGUoY29sb3IgPSAid2hpdGUiKSsKICBzY2FsZV9maWxsX2dyYWRpZW50Mihsb3cgPSAiYmx1ZSIsIGhpZ2ggPSAicmVkIiwgbWlkID0gIndoaXRlIiwgbWlkcG9pbnQgPSAwLCBsaW1pdCA9IGMoLTEsMSksIHNwYWNlID0gIkxhYiIsIG5hbWU9IlBlYXJzb25cbkNvcnJlbGF0aW9uIikgKwogIHRoZW1lX21pbmltYWwoKSsKICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCB2anVzdCA9IDEsIHNpemUgPSAxMiwgaGp1c3QgPSAxKSkrCiAgY29vcmRfZml4ZWQoKSsKICBsYWJzKHRpdGxlID0gJ1JlZCBXaW5lJykKCmBgYAoKCgoKCmBgYHtyfQoKCmdncGxvdCh0cmFpbl93aXRoX3F1YWwsIG1hcHBpbmcgPSBhZXMoeD1xdWFsaXR5LCBmaWxsPVR5cGUpKSsKICBnZW9tX2hpc3RvZ3JhbShiaW53aWR0aD0xLCBhbHBoYT0uNCwgcG9zaXRpb249ImlkZW50aXR5IiwgY29sb3I9ImJsYWNrIikrCiAgZ2VvbV92bGluZShhZXMoeGludGVyY2VwdD01LjUsIGNvbG9yPSJyZWQiKSwKICAgICAgICAgICAgIGxpbmV0eXBlPSJkYXNoZWQiKSsKICBzY2FsZV9jb2xvcl9tYW51YWwobmFtZSA9ICJDdXQgT2ZmIiwgdmFsdWVzID0gYygicmVkIikpKwogIGxhYnMoeD0iUXVhbGl0eSIsCiAgICAgICB5PSJGcmVxdWVuY3kiLAogICAgICAgdGl0bGU9IkRpc3RyaWJ1dGlvbiBvZiBRdWFsaXR5IFJhdGluZyBieSBXaW5lIFR5cGUiKQpgYGAKCgoKVGhpcyBpcyB0aGUgdGFibGUgZm9yIHNob3dpbmcgdGhlIGV2YWx1YXRpb24gZm9yIHRoZSBmaXJzdCBtb2RlbApgYGB7cn0KZm9ybWF0dGFibGUoZXZ1bGF0aW9uX3N1bW1hcnlbMSxdKQpgYGAKCgoKYGBge3J9CnN1bW1hcnkobW9kZWwxX1JlZCkKYGBgCgpgYGB7cn0Kc3VtbWFyeShtb2RlbDFfV2hpdGUpCmBgYAoKCmBgYHtyfQpzdW1tYXJ5KG1vZGVsMV9pbnQpCmBgYAoKCgoKVGFibGUgcmlnaHQgYWJvdmUgdGhlICJCZXN0IFBvc3NpYmxlIE1vZGVsIChSZWR1Y2VkXzQpIiBzZWN0aW9uLgpgYGB7cn0KZm9ybWF0dGFibGUoY29tcGFyZV9tb2RlbHMsYWxpZ24gPWMoImwiLCJjIiwgImMiLCAiYyIsICJjIiwgInIiKSkKCmBgYAoKCgoKVGhpcyBpcyB0aGUgdGFibGUgZm9yIHNob3dpbmcgdGhlIGJlc3QgbW9kZWxzICh0b3AgZml2ZSkKYGBge3J9CmZvcm1hdHRhYmxlKHJlcy5iZXN0LmxvZ2lzdGljJEJlc3RNb2RlbHMpCmBgYAoKCkFkZCBST0MgZm9yIHJlZHVjZWRfMSwgbW9kZWwxX1JlZCwgbW9kZWwxX1doaXRlLCBtb2RlbDFfaW50CgpgYGB7cn0KCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfMSxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfMSIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzwtcGVyZm9ybWFuY2UocmF0ZXMsIG1lYXN1cmUgPSAiYXVjIikKcmVkdWNlZF8xX2F1YyA8LSBhdWNAeS52YWx1ZXMKCmBgYAoKCmBgYHtyfQojIG1vZGVsMV9SZWQKcHJlZHM8LXByZWRpY3QobW9kZWwxX1JlZCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9SZWQiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCm1vZGVsMV9SZWRfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKYGBgCgpgYGB7cn0KCiMgbW9kZWwxX1doaXRlCnByZWRzPC1wcmVkaWN0KG1vZGVsMV9XaGl0ZSxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9XaGl0ZSIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzwtcGVyZm9ybWFuY2UocmF0ZXMsIG1lYXN1cmUgPSAiYXVjIikKbW9kZWwxX1doaXRlX2F1YyA8LSBhdWNAeS52YWx1ZXMKCmBgYAoKYGBge3J9CgojIG1vZGVsMV9pbnQKcHJlZHM8LXByZWRpY3QobW9kZWwxX2ludCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIG1vZGVsMV9pbnQiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCm1vZGVsMV9pbnRfYXVjIDwtIGF1Y0B5LnZhbHVlcwpgYGAKCgpUaGlzIGlzIHRoZSB0YWJsZSBmb3Igc2hvd2luZyB0aGUgYmVzdCBtb2RlbHMgKHRvcCBmaXZlKQpgYGB7cn0KZm9ybWF0dGFibGUoZXZ1bGF0aW9uX3N1bW1hcnlbMixdKQpgYGAKCgpUaGlzIGlzIHRoZSB0YWJsZSBmb3IgcmVkdWNlZF80IFZJRi4KYGBge3J9CmZvcm1hdHRhYmxlKGRhdGEuZnJhbWUocmVnXzRfVklGX3Rlc3QpLCBhbGlnbiA9YygibCIsInIiKSkKYGBgCgpUaGlzIGlzIHRoZSBuZXh0IFZJRiBwbG90IGluIHRoZSByZXBvcnQKYGBge3J9CmZvcm1hdHRhYmxlKGRhdGEuZnJhbWUocmVnXzRfMl9WSUZfdGVzdCksIGFsaWduID1jKCJsIiwiciIpKQpgYGAKCgpUaGUgdGFibGUgYmVsb3cgdGhhdC4gSXQgaXMgdGhlIGV2YWx1YXRpb24gc3VtbWFyeSBmb3IgcmVkdWNlZF80XzIKYGBge3J9CmZvcm1hdHRhYmxlKGV2dWxhdGlvbl9zdW1tYXJ5WzMsXSkKYGBgCgoKZXZhbHVhdGlvbiBzdW1tYXJ5IGZvciB0aGUgb3V0bGllci9sZXZlcmFnZS9ldGMuCmBgYHtyfQpmb3JtYXR0YWJsZShldnVsYXRpb25fc3VtbWFyeVs0OjcsXSkKYGBgCgoKCmFkZCByb2MgY3VydmVzIGZvciB0aGVzZSBmb3VyLgoKYGBge3J9CgoKIyMgcmVkdWNlZF80CnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM0PC1wcmVkaWN0aW9uKHByZWRzLCB0ZXN0JGNhdF9xdWFsaXR5KQpyb2NfcmVzdWx0PC1wZXJmb3JtYW5jZShyYXRlczQsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQpwbG90KHJvY19yZXN1bHQsIG1haW49IlJPQyBDdXJ2ZSBmb3IgcmVkdWNlZF80IikKbGluZXMoeCA9IGMoMCwxKSwgeSA9IGMoMCwxKSwgY29sPSJyZWQiKQoKYXVjNDwtcGVyZm9ybWFuY2UocmF0ZXM0LCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF9hdWMgPC0gYXVjNEB5LnZhbHVlcwoKYGBgCgpgYGB7cn0KCiMjIHJlZHVjZWRfNF8yCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF8yLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQpwbG90KHJvY19yZXN1bHQsIG1haW49IlJPQyBDdXJ2ZSBmb3IgcmVkdWNlZF80XzIiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM0XzI8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF8yX2F1YyA8LSBhdWM0XzJAeS52YWx1ZXMKCmBgYAoKCmBgYHtyfQoKIyMgcmVkdWNlZF80XzMKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzMsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCnJhdGVzPC1wcmVkaWN0aW9uKHByZWRzLCB0ZXN0JGNhdF9xdWFsaXR5KQpyb2NfcmVzdWx0PC1wZXJmb3JtYW5jZShyYXRlcyxtZWFzdXJlPSJ0cHIiLCB4Lm1lYXN1cmU9ImZwciIpCnBsb3Qocm9jX3Jlc3VsdCwgbWFpbj0iUk9DIEN1cnZlIGZvciByZWR1Y2VkXzRfMyIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzwtcGVyZm9ybWFuY2UocmF0ZXMsIG1lYXN1cmUgPSAiYXVjIikKcmVkdWNlZF80XzNfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKCmBgYAoKCmBgYHtyfQojIyByZWR1Y2VkXzRfNF9sZXYgCnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfNF80X2xldixuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNF80X2xldiIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzwtcGVyZm9ybWFuY2UocmF0ZXMsIG1lYXN1cmUgPSAiYXVjIikKcmVkdWNlZF80XzRfbGV2X2F1YyA8LSBhdWNAeS52YWx1ZXMKCmBgYAoKYGBge3J9CgojIyByZWR1Y2VkXzRfNV9EREZGSVRTIApwcmVkczwtcHJlZGljdChyZWR1Y2VkXzRfNV9EREZGSVRTLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRjYXRfcXVhbGl0eSkKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQpwbG90KHJvY19yZXN1bHQsIG1haW49IlJPQyBDdXJ2ZSBmb3IgcmVkdWNlZF80XzVfRERGRklUUyIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKCmF1YzwtcGVyZm9ybWFuY2UocmF0ZXMsIG1lYXN1cmUgPSAiYXVjIikKcmVkdWNlZF80XzVfRERGRklUU19hdWMgPC0gYXVjQHkudmFsdWVzCgpgYGAKCgoKCmBgYHtyfQoKIyMgcmVkdWNlZF80XzZfbm9fc3BlY2lhbCAKcHJlZHM8LXByZWRpY3QocmVkdWNlZF80XzZfbm9fc3BlY2lhbCxuZXdkYXRhPXRlc3QsIHR5cGU9InJlc3BvbnNlIikKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkY2F0X3F1YWxpdHkpCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIHJlZHVjZWRfNF82X25vX3NwZWNpYWwiKQpsaW5lcyh4ID0gYygwLDEpLCB5ID0gYygwLDEpLCBjb2w9InJlZCIpCgphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCnJlZHVjZWRfNF82X25vX3NwZWNpYWxfYXVjIDwtIGF1Y0B5LnZhbHVlcwoKYGBgCgpgYGB7cn0KQVVDX3N1bW1hcnkgPC0gZGF0YS5mcmFtZSgncmVkdWNlZF8xJz1yZWR1Y2VkXzFfYXVjLAogICAgICAgICAgICAgICAgICAgICAgICAgICdyZWR1Y2VkXzQnPXJlZHVjZWRfNF9hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF8yJz1yZWR1Y2VkXzRfMl9hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF8zJz1yZWR1Y2VkXzRfM19hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF80X2xldic9cmVkdWNlZF80XzRfbGV2X2F1YywKICAgICAgICAgICAgICAgICAgICAgICAgICAncmVkdWNlZF80XzVfRERGRklUUyc9cmVkdWNlZF80XzVfRERGRklUU19hdWMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgJ3JlZHVjZWRfNF82X25vX3NwZWNpYWwnPXJlZHVjZWRfNF82X25vX3NwZWNpYWxfYXVjKQpjb2xuYW1lcyhBVUNfc3VtbWFyeSkgPC0gYygncmVkdWNlZF8xJywncmVkdWNlZF80JywncmVkdWNlZF80XzInLCdyZWR1Y2VkXzRfMycsJ3JlZHVjZWRfNF80X2xldicsJ3JlZHVjZWRfNF81X0RERkZJVFMnLCdyZWR1Y2VkXzRfNl9ub19zcGVjaWFsJykKCkFVQ19zdW1tYXJ5CmBgYAoKCmBgYHtyfQpmb3JtYXR0YWJsZShldnVsYXRpb25fc3VtbWFyeVsxOjcsXSkKYGBgCgoKCgoKCgoKCgoKCiMjIEZpcnN0IG1vZGVsIGFuZCBhZGRpdGlvbmFsIEVEQQoKYGBge3Igc2V0dXAsIGluY2x1ZGU9RkFMU0V9CmtuaXRyOjpvcHRzX2NodW5rJHNldChlY2hvID0gVFJVRSkKYGBgCgpgYGB7ciBpbmNsdWRlPUZBTFNFfQpsaWJyYXJ5KFJPQ1IpCmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeShNQVNTKQpsaWJyYXJ5KGZhcmF3YXkpCmBgYAoKCmBgYHtyIGluY2x1ZGU9RkFMU0V9CiMjIExvYWQgRGF0YXNldHMKZnVsbF93aW5lc19maW5hbCA8LSByZWFkLmNzdigiRGF0YV9GaW5hbC5jc3YiLCBoZWFkZXIgPSBUUlVFLCBzdHJpbmdzQXNGYWN0b3JzPVRSVUUpCiMgRHJvcCBxdWFsaXR5IGZvciBzaW1wbGljaXR5CmZ1bGxfd2luZXNfYmluYXJ5IDwtIHN1YnNldChmdWxsX3dpbmVzX2ZpbmFsLCBzZWxlY3QgPSAtYyhjYXRfcXVhbGl0eSkpCgpzZXQuc2VlZCg5MDIxMCkgIyNmb3IgcmVwcm9kdWNpYmlsaXR5CnNhbXBsZTwtc2FtcGxlLmludChucm93KGZ1bGxfd2luZXNfYmluYXJ5KSwgZmxvb3IoLjgwKm5yb3coZnVsbF93aW5lc19iaW5hcnkpKSwgcmVwbGFjZSA9IEYpCnRyYWluPC1mdWxsX3dpbmVzX2JpbmFyeVtzYW1wbGUsIF0gIyN0cmFpbmluZyBkYXRhIGZyYW1lCnRlc3Q8LWZ1bGxfd2luZXNfYmluYXJ5Wy1zYW1wbGUsIF0gIyN0ZXN0IGRhdGEgZnJhbWUKYGBgCgpUaGUgZmlyc3QgcmVzZWFyY2ggcXVlc3Rpb24gZm9jdXNlZCBvbiBleHBsb3JpbmcgdGhlIHJlbGF0aW9uc2hpcHMgYmV0d2VlbiBkaWZmZXJlbnQgY2hlbWljYWwgZmVhdHVyZXMgaW4gdGhlIGRhdGFzZXQgYW5kIHdpbmUgdHlwZSAoY2F0ZWdvcml6ZWQgYXMgZWl0aGVyIHJlZCBvciB3aGl0ZSkuIFNpbmNlIHRoZSByZXNwb25zZSB2YXJpYWJsZSwgd2luZSB0eXBlLCB3YXMgYSBCZXJub3VsbGkgcmFuZG9tIHZhcmlhYmxlIGFuZCB0aGUgb2JzZXJ2YXRpb25zIHdlcmUgaW5kZXBlbmRlbnQsIGEgbG9naXN0aWMgcmVncmVzc2lvbiB3YXMgdXNlZCBmb3IgdGhlIHByZWRpY3RpdmUgbW9kZWwuCgpBcyBzZWVuIGFib3ZlIGluIHRoZSBwcmV2aW91cyBzZWN0aW9uLCBleHBsb3JhdG9yeSBkYXRhIGFuYWx5c2lzIHdhcyBwZXJmb3JtZWQgaW4gb3JkZXIgdG8gYmV0dGVyIHVuZGVyc3RhbmQgdGhlIHJlbGF0aW9uc2hpcHMgYmV0d2VlbiBpbmRpdmlkdWFsIHByZWRpY3RvciB2YXJpYWJsZXMgYW5kIHdpbmUgdHlwZS4gRGVuc2l0eSBwbG90cyB2aXN1YWxpemVkIHRoZSBxdWFudGl0YXRpdmUgcHJlZGljdG9ycywgd2hpbGUgZnJlcXVlbmN5IGJhciBjaGFydHMgdmlzdWFsaXplZCB0aGUgY2F0ZWdvcmljYWwgd2luZSBxdWFsaXR5IHByZWRpY3Rvci4gVGhlIG1ham9yaXR5IG9mIHRoZXNlIGZlYXR1cmVzIHdlcmUgZGlzdHJpYnV0ZWQgZGlmZmVyZW50bHkgYWNyb3NzIHdpbmUgdHlwZSwgaW5kaWNhdGluZyB0aGF0IHRoZXkgY291bGQgYmUgdXNlZCB0byBkaWZmZXJlbnRpYXRlIGJldHdlZW4gcmVkIGFuZCB3aGl0ZSB3aW5lIGluIHRoZSBtb2RlbC4gSG93ZXZlciwgYXMgc2VlbiBpbiBGaWd1cmUgWFhYLCBhbGNvaG9sLCBwSCwgYW5kIHN1bHBoYXRlcyBkbyBzZWVtIHRvIGhhdmUgYSB3ZWFrZXIgYXNzb2NpYXRpb24gd2l0aCB3aW5lIHR5cGUgc2luY2UgdGhlaXIgcmVkIGFuZCB3aGl0ZSBkZW5zaXR5IHBsb3RzIGxvb2sgc29tZXdoYXQgc2ltaWxhci4KYGBge3IgZWNobz1GQUxTRX0KZ2dwbG90KHRyYWluLCBhZXMoeD1UeXBlKSkrCiAgZ2VvbV9iYXIoKSsKICBsYWJzKHg9IldpbmUgVHlwZSIsIHk9ImNvdW50IiwKICAgICAgIHRpdGxlPSJXaW5lIFR5cGUiKQoKZ2dwbG90KHRyYWluLCBhZXMoeD1xdWFsaXR5LCBmaWxsPVR5cGUpKSsKICBnZW9tX2Jhcihwb3NpdGlvbiA9ICJmaWxsIikrCiAgbGFicyh4PSJXaW5lIFF1YWxpdHkiLCB5PSJQcm9wb3J0aW9uIiwKICAgICAgIHRpdGxlPSJXaW5lIFF1YWxpdHkgYnkgVHlwZSIpCgojI2RlbnNpdHkgcGxvdHMKZ2dwbG90KHRyYWluLGFlcyh4PXZvbGF0aWxlLmFjaWRpdHksIGNvbG9yPVR5cGUpKSsKICBnZW9tX2RlbnNpdHkoKSsKICBsYWJzKHRpdGxlPSJEZW5zaXR5IFBsb3Qgb2YgVm9sYXRpbGUgQWNpZGl0eSBieSBUeXBlIikKCmdncGxvdCh0cmFpbixhZXMoeD1jaXRyaWMuYWNpZCwgY29sb3I9VHlwZSkpKwogIGdlb21fZGVuc2l0eSgpKwogIGxhYnModGl0bGU9IkRlbnNpdHkgUGxvdCBvZiBDaXRyaWMgQWNpZCBieSBUeXBlIikKCmdncGxvdCh0cmFpbixhZXMoeD1yZXNpZHVhbC5zdWdhciwgY29sb3I9VHlwZSkpKwogIGdlb21fZGVuc2l0eSgpKwogIGxhYnModGl0bGU9IkRlbnNpdHkgUGxvdCBvZiBSZXNpZHVhbCBTdWdhcnMgYnkgVHlwZSIpCgpnZ3Bsb3QodHJhaW4sYWVzKHg9Y2hsb3JpZGVzLCBjb2xvcj1UeXBlKSkrCiAgZ2VvbV9kZW5zaXR5KCkrCiAgbGFicyh0aXRsZT0iRGVuc2l0eSBQbG90IG9mIENobG9yaWRlcyBTdWdhcnMgYnkgVHlwZSIpCgpnZ3Bsb3QodHJhaW4sYWVzKHg9ZnJlZS5zdWxmdXIuZGlveGlkZSwgY29sb3I9VHlwZSkpKwogIGdlb21fZGVuc2l0eSgpKwogIGxhYnModGl0bGU9IkRlbnNpdHkgUGxvdCBvZiBGcmVlIFN1bGZ1ciBEaW94aWRlIGJ5IFR5cGUiKQoKZ2dwbG90KHRyYWluLGFlcyh4PXRvdGFsLnN1bGZ1ci5kaW94aWRlLCBjb2xvcj1UeXBlKSkrCiAgZ2VvbV9kZW5zaXR5KCkrCiAgbGFicyh0aXRsZT0iRGVuc2l0eSBQbG90IG9mIFRvdGFsIFN1bGZ1ciBEaW94aWRlIGJ5IFR5cGUiKQoKZ2dwbG90KHRyYWluLGFlcyh4PWRlbnNpdHksIGNvbG9yPVR5cGUpKSsKICBnZW9tX2RlbnNpdHkoKSsKICBsYWJzKHRpdGxlPSJEZW5zaXR5IFBsb3Qgb2YgRGVuc2l0eSBieSBUeXBlIikKCmdncGxvdCh0cmFpbixhZXMoeD1wSCwgY29sb3I9VHlwZSkpKwogIGdlb21fZGVuc2l0eSgpKwogIGxhYnModGl0bGU9IkRlbnNpdHkgUGxvdCBvZiBwSCBieSBUeXBlIikKCmdncGxvdCh0cmFpbixhZXMoeD1zdWxwaGF0ZXMsIGNvbG9yPVR5cGUpKSsKICBnZW9tX2RlbnNpdHkoKSsKICBsYWJzKHRpdGxlPSJEZW5zaXR5IFBsb3Qgb2YgU3VscGhhdGVzIGJ5IFR5cGUiKQoKZ2dwbG90KHRyYWluLGFlcyh4PWFsY29ob2wsIGNvbG9yPVR5cGUpKSsKICBnZW9tX2RlbnNpdHkoKSsKICBsYWJzKHRpdGxlPSJEZW5zaXR5IFBsb3Qgb2YgQWxjb2hvbCBieSBUeXBlIikKYGBgCgpBZnRlciBleHBsb3JpbmcgdGhlc2UgZmVhdHVyZXMsIGFuIGluaXRpYWwgbW9kZWwgd2FzIGdlbmVyYXRlZCB3aXRoIGFsbCBvZiB0aGUgcHJlZGljdG9ycyB1c2luZyB0aGUgcmFuZG9tIHRyYWluaW5nIHNhbXBsZSBmcm9tIHRoZSBjb21iaW5lZCB3aW5lIGRhdGFzZXQuIEZyb20gdGhlIG1vZGVsIHN1bW1hcnksIHRoZSBjaGVtaWNhbCBwcm9wZXJ0aWVzIG9mIGZpeGVkIGFjaWRpdHksIGNpdHJpYyBhY2lkLCBwSCwgYW5kIHN1bHBoYXRlcyBhbGwgYXBwZWFyZWQgdG8gaGF2ZSBpbnNpZ25pZmljYW50IHotc2NvcmVzLiBUbyBkZXRlcm1pbmUgaWYgYWxsIG9mIHRoZXNlIGZlYXR1cmVzIGNvdWxkIGJlIGRyb3BwZWQgZnJvbSB0aGUgbW9kZWwsIGEgbGlrZWxpaG9vZCByYXRpbyB0ZXN0IGNvbXBhcmluZyB0aGUgZnVsbCBhbmQgcmVkdWNlZCBtb2RlbCB3YXMgY29uZHVjdGVkLiBUaGlzIHRlc3QgcmVzdWx0ZWQgaW4gYW4gaW5zaWduaWZpY2FudCBwLXZhbHVlIG9mIDAuMTM2MC4gVGhlcmVmb3JlLCB0aGVyZSB3YXMgbm90IGEgc2lnbmlmaWNhbnQgZGlmZmVyZW5jZSBiZXR3ZWVuIHRoZSB0d28gbW9kZWxzIGFuZCB0aGUgcmVkdWNlZCBtb2RlbCBjb3VsZCBiZSBhZG9wdGVkLiBUaGlzIG1vZGVs4oCZcyBwZXJmb3JtYW5jZSBpcyBzdW1tYXJpemVkIGJlbG93LiBYWFgKYGBge3IgZWNobz1GQUxTRX0KbG9nX21vZDwtIGdsbShUeXBlIH4uLCBmYW1pbHk9J2Jpbm9taWFsJywgZGF0YT10cmFpbikKc3VtbWFyeShsb2dfbW9kKQpgYGAKCmBgYHtyIGVjaG89RkFMU0V9CnByZWRzPC1wcmVkaWN0KGxvZ19tb2QsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCgpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRUeXBlKQoKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQoKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIEZ1bGwgV2luZSBUeXBlIE1vZGVsIikKbGluZXMoeCA9IGMoMCwxKSwgeSA9IGMoMCwxKSwgY29sPSJyZWQiKQpgYGAKCmBgYHtyIGVjaG89RkFMU0V9CmF1YzwtcGVyZm9ybWFuY2UocmF0ZXMsIG1lYXN1cmUgPSAiYXVjIikKYXVjQHkudmFsdWVzCmBgYAoKYGBge3IgZWNobz1GQUxTRX0KdGFibGUodGVzdCRUeXBlLCBwcmVkcz4wLjY1KQpgYGAKCmBgYHtyIGVjaG89RkFMU0V9CnJlZHVjZWRfbW9kPC0gZ2xtKFR5cGUgfi4gLXF1YWxpdHktcEgtZml4ZWQuYWNpZGl0eS1jaXRyaWMuYWNpZCwgZmFtaWx5PSdiaW5vbWlhbCcsIGRhdGE9dHJhaW4pCnN1bW1hcnkocmVkdWNlZF9tb2QpCmBgYAoKYGBge3IgaW5jbHVkZT1GQUxTRX0KZyA8LSByZWR1Y2VkX21vZCRkZXZpYW5jZS1sb2dfbW9kJGRldmlhbmNlCjEtcGNoaXNxKGcsNCkKYGBgCgpUaGUgUk9DIGN1cnZlIFhYWCwgc2VlbiBiZWxvdywgaW5kaWNhdGVzIHRoYXQgdGhlIG1vZGVsIHBlcmZvcm1lZCB3ZWxsIChzaWduaWZpY2FudGx5IGJldHRlciB0aGFuIHJhbmRvbSBndWVzc2luZyBzaG93biBieSB0aGUgcmVkIGxpbmUpLiBGdXJ0aGVybW9yZSwgdGhlIEFVQywgd2hpY2ggcmVwcmVzZW50cyB0aGUgYXJlYSB1bmRlciB0aGlzIGN1cnZlLCB3YXMgMC45OTQuIFRoZXNlIHBlcmZvcm1hbmNlIG1ldHJpY3Mgc3VnZ2VzdCB0aGF0IHRoZSBtb2RlbCBpcyBxdWl0ZSBlZmZlY3RpdmUgYXQgcHJlZGljdGluZyB0aGUgd2luZSB0eXBlIG9mIHRoZSB0cmFpbmluZyBkYXRhLiBXaGVuIHRoaXMgbW9kZWwgcHJlZGljdGVkIHRoZSB3aW5lIHR5cGUgb2YgdGhlIHRlc3RpbmcgZGF0YSAodGhlIHNhbXBsZSBsZWZ0b3ZlciBmcm9tIHRoZSB0cmFpbmluZyBzZXQpLCBpdCBtYWludGFpbmVkIHRoaXMgaGlnaCBwZXJmb3JtYW5jZS4gV2UgdXNlZCBhIHByb2JhYmlsaXR5IGZsb29yIG9mIDAuNjUgc2luY2UgdGhpcyB2YWx1ZSBtaW5pbWl6ZWQgb3ZlcmFsbCBlcnJvciAod2Ugc2F3IG5vIHByYWN0aWNhbCBkaWZmZXJlbmNlIGJldHdlZW4gdHlwZSBJIGFuZCB0eXBlIElJIGVycm9ycyBpbiB0aGlzIGNvbnRleHQpLiBVc2luZyB0aGlzIHZhbHVlLCB0aGUgbW9kZWwgaGFkIGFuIG92ZXJhbGwgZXJyb3IgcmF0ZSBvZiBvbmx5IDAuMDA2Mi4gVGhpcyByZXN1bHQgZGVtb25zdHJhdGVzIHRoZSBlZmZlY3RpdmVuZXNzIG9mIHRoZSBtb2RlbCB3aGVuIHByZXNlbnRlZCB3aXRoIGRhdGEgaXQgaGFkIG5vdCBlbmNvdW50ZXJlZCBpbiB0cmFpbmluZy4gCmBgYHtyIGVjaG89RkFMU0V9CnByZWRzPC1wcmVkaWN0KHJlZHVjZWRfbW9kLG5ld2RhdGE9dGVzdCwgdHlwZT0icmVzcG9uc2UiKQoKcmF0ZXM8LXByZWRpY3Rpb24ocHJlZHMsIHRlc3QkVHlwZSkKCnJvY19yZXN1bHQ8LXBlcmZvcm1hbmNlKHJhdGVzLG1lYXN1cmU9InRwciIsIHgubWVhc3VyZT0iZnByIikKCnBsb3Qocm9jX3Jlc3VsdCwgbWFpbj0iUk9DIEN1cnZlIGZvciBSZWR1Y2VkIFdpbmUgVHlwZSBNb2RlbCIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKYGBgCgpgYGB7ciBpbmNsdWRlPUZBTFNFfQphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCmF1Y0B5LnZhbHVlcwpgYGAKCgpgYGB7ciBlY2hvPUZBTFNFfQp0YWJsZSh0ZXN0JFR5cGUsIHByZWRzPjAuNjUpCmBgYAoKVG8gZnVydGhlciBpbXByb3ZlIHRoZSBtb2RlbOKAmXMgcGVyZm9ybWFuY2UsIHRoZSBwcmVzZW5jZSBvZiBvdXRsaWVycyBpbiB0aGUgdHJhaW5pbmcgZGF0YSB3YXMgY29uc2lkZXJlZC4gVXNpbmcgQ29va+KAmXMgZGlzdGFuY2UsIG9uZSBvdXRsaWVyICh3aXRoIGEgZGlzdGFuY2Ugb2YgMy4wODU4KSB3YXMgaWRlbnRpZmllZC4gSG93ZXZlciwgYWZ0ZXIgcmVmaXR0aW5nIGEgbW9kZWwgd2l0aG91dCB0aGlzIG91dGxpZXIsIHRoZXJlIHdhcyBhIHZlcnkgc21hbGwgZGlmZmVyZW5jZSBpbiBvdmVyYWxsIHBlcmZvcm1hbmNlIGFzIHNlZW4gYmVsb3cgWFhYLiBUaHVzLCB3ZSBlbGVjdGVkIHRvIGtlZXAgdGhlIG91dGxpZXIgaW4gdGhlIGRhdGHigJRlc3BlY2lhbGx5LCBzaW5jZSB3ZSBoYWQgbGl0dGxlIGluc2lnaHQgaW50byB0aGUgZGF0YSBhbmQgY291bGQgbm90IGRlZHVjZSB3aGV0aGVyIGl0IHdhcyBhIGNvcnJ1cHRlZCBkYXRhcG9pbnQuCmBgYHtyIGVjaG89RkFMU0V9Cm48LWRpbSh0cmFpbilbMV0KcDwtOQoKIyNjb29rcyBkaXN0YW5jZQpvdXRsaWVyPC0gdHJhaW4Kb3V0bGllclsnQ09PS1MnXTwtIGNvb2tzLmRpc3RhbmNlKHJlZHVjZWRfbW9kKQpvdXRsaWVyW291dGxpZXIkQ09PS1M+cWYoMC41LHAsbi1wKSxdCmBgYAoKYGBge3IgaW5jbHVkZT1GQUxTRX0KbmV3X2RhdGE8LW91dGxpZXJbIW91dGxpZXIkQ09PS1M+cWYoMC41LHAsbi1wKSwtMTRdCm5ld19mdWxsX21vZDwtIGdsbShUeXBlIH4uLCBmYW1pbHk9J2Jpbm9taWFsJywgZGF0YT1uZXdfZGF0YSkKc3VtbWFyeShuZXdfZnVsbF9tb2QpCmBgYApgYGB7ciBlY2hvPUZBTFNFfQpuZXdfcmVkdWNlZF9tb2Q8LSBnbG0oVHlwZSB+LiAtcXVhbGl0eS1zdWxwaGF0ZXMtY2l0cmljLmFjaWQsIGZhbWlseT0nYmlub21pYWwnLCBkYXRhPW5ld19kYXRhKQpzdW1tYXJ5KG5ld19yZWR1Y2VkX21vZCkKYGBgCgpgYGB7ciBlY2hvPUZBTFNFfQpwcmVkczwtcHJlZGljdChuZXdfcmVkdWNlZF9tb2QsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCgpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRUeXBlKQoKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQoKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIFdpbmUgVHlwZSIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKYGBgCgpgYGB7ciBpbmNsdWRlPUZBTFNFfQphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCmF1Y0B5LnZhbHVlcwpgYGAKCmBgYHtyIGVjaG89RkFMU0V9CnRhYmxlKHRlc3QkVHlwZSwgcHJlZHM+MC40KQpgYGAKCkZpbmFsbHksIGNvbGxpbmVhcml0eSBhbW9uZyB0aGUgcHJlZGljdG9ycyB3YXMgZXhhbWluZWQgaW4gdGhlIHJlZHVjZWQgbW9kZWwuIFVzaW5nIHRoZSBWSUYgZm9ybXVsYSwgZGVuc2l0eSB3YXMgdmVyeSBjb3JyZWxhdGVkIHdpdGggdGhlIG90aGVyIHByZWRpY3RvcnMgKGhhdmluZyBhIFZJRiB2YWx1ZSBvZiAxMy43MTQpLiBTaW5jZSB0aGlzIHZhbHVlIHdhcyBvdmVyIDEwLCB0aGlzIHBhcmFtZXRlciB3YXMgZHJvcHBlZCBpbiBvcmRlciB0byByZWR1Y2UgY29sbGluZWFyaXR5IGFuZCBieSBleHRlbnNpb24gdGhlIHByZWRpY3RlZCBjb2VmZmljaWVudHPigJkgdmFyaWFuY2VzLiBJbiBhZGRpdGlvbiwgdGhlIGNvcnJlbGF0aW9uIG1hdHJpeCBvZiB0aGUgcHJlZGljdG9ycyBzaG93ZWQgYSBzdHJvbmcgYXNzb2NpYXRpb24gYmV0d2VlbiBmcmVlIHN1bGZ1ciBkaW94aWRlIGFuZCB0b3RhbCBzdWxmdXIgZGlveGlkZSAoY29ycmVsYXRpb24gb2YgMC43MTU2KS4gU2luY2Ugb25lIG9mIHRoZXNlIGZlYXR1cmVzIGNvdWxkIGV4cGxhaW4gbXVjaCBvZiB0aGUgdmFyaWF0aW9uIGluIHRoZSBvdGhlciwgaXQgd2FzIGRldGVybWluZWQgb25lIHNob3VsZCBiZSBkcm9wcGVkLiBXaGVuIGNvbXBhcmluZyBtb2RlbHMgd2l0aCBvbmx5IG9uZSBvZiB0aGVzZSBmZWF0dXJlcywgaXQgd2FzIGV2aWRlbnQgdGhhdCB0b3RhbCBzdWxmdXIgZGlveGlkZSB3YXMgbW9yZSBpbXBvcnRhbnQgaW4gcHJlZGljdGluZyB3aW5lIHR5cGUgdGhhbiBmcmVlIHN1bGZ1ciBkaW94aWRlLiBUaGVyZWZvcmUsIGZyZWUgc3VsZnVyIGRpb3hpZGUgd2FzIGRyb3BwZWQgZnJvbSB0aGUgbW9kZWwuIFRoZSBwZXJmb3JtYW5jZSBvZiB0aGlzIG5ldyByZWR1Y2VkIG1vZGVsIGNhbiBiZSBzZWVuIGJlbG93IFhYWC4KYGBge3IgZWNobz1GQUxTRX0KeDwtIHRyYWluWyxjKDEsMiw0LDUsNiw3LDgsOSwxMSldCnZpZih4KQpgYGAKYGBge3IgZWNobz1GQUxTRX0KY29yKHgpCmBgYAoKYGBge3IgZWNobz1GQUxTRX0KcmVkdWNlZF9tb2RfZmluYWw8LSBnbG0oVHlwZSB+LiAtcXVhbGl0eS1wSC1maXhlZC5hY2lkaXR5LWNpdHJpYy5hY2lkLWRlbnNpdHktZnJlZS5zdWxmdXIuZGlveGlkZSwgZmFtaWx5PSdiaW5vbWlhbCcsIGRhdGE9dHJhaW4pCnN1bW1hcnkocmVkdWNlZF9tb2RfZmluYWwpCmBgYAoKYGBge3IgZWNobz1GQUxTRX0KcHJlZHM8LXByZWRpY3QocmVkdWNlZF9tb2RfZmluYWwsbmV3ZGF0YT10ZXN0LCB0eXBlPSJyZXNwb25zZSIpCgpyYXRlczwtcHJlZGljdGlvbihwcmVkcywgdGVzdCRUeXBlKQoKcm9jX3Jlc3VsdDwtcGVyZm9ybWFuY2UocmF0ZXMsbWVhc3VyZT0idHByIiwgeC5tZWFzdXJlPSJmcHIiKQoKcGxvdChyb2NfcmVzdWx0LCBtYWluPSJST0MgQ3VydmUgZm9yIEZpbmFsIFdpbmUgVHlwZSBNb2RlbCIpCmxpbmVzKHggPSBjKDAsMSksIHkgPSBjKDAsMSksIGNvbD0icmVkIikKYGBgCgpgYGB7ciBpbmNsdWRlPUZBTFNFfQphdWM8LXBlcmZvcm1hbmNlKHJhdGVzLCBtZWFzdXJlID0gImF1YyIpCmF1Y0B5LnZhbHVlcwpgYGAKCmBgYHtyIGVjaG89RkFMU0V9CnRhYmxlKHRlc3QkVHlwZSwgcHJlZHM+MC44KQpgYGAKClRob3VnaCB0aGUgbW9kZWzigJlzIHBlcmZvcm1hbmNlIGRlY3JlYXNlZCBzbGlnaHRseSAob3ZlcmFsbCBlcnJvciByYXRlIHdlbnQgZnJvbSAwLjAwNjIgdG8gMC4wMjk3IGFuZCBBVUMgd2VudCBmcm9tIDAuOTk0NyB0byAwLjk4OTIpLCB0aGUgZGVjcmVhc2UgaW4gdmFyaWFuY2Ugb2YgdGhlIGNvZWZmaWNpZW50cyBzaG91bGQgbWFrZSB0aGUgbW9kZWwgbW9yZSByZWxpYWJsZSBpbiB0ZXJtcyBvZiBpbnRlcnByZXRpbmcgdGhlIHJlbGF0aW9uc2hpcHMgYmV0d2VlbiB0aGUgcHJlZGljdG9ycyBhbmQgd2luZSB0eXBlLgoKT3ZlcmFsbCwgdGhpcyBmaW5hbCBtb2RlbCBpcyBxdWl0ZSBlZmZlY3RpdmUgYXQgZGV0ZXJtaW5pbmcgd2luZSB0eXBlLiBXaGVuIGxvb2tpbmcgYXQgdGhlIGVzdGltYXRlZCBjb2VmZmljaWVudHMsIGl0IHNlZW1zIGFzIHRob3VnaCBoaWdoZXIgcmVzaWR1YWwgc3VnYXIsIHRvdGFsIHN1bGZ1ciBkaW94aWRlLCBhbmQgYWxjb2hvbCBhcmUgYXNzb2NpYXRlZCB3aXRoIHdoaXRlIHdpbmVzLCB3aGlsZSBoaWdoZXIgdm9sYXRpbGUgYWNpZGl0eSwgY2hsb3JpZGVzLCBhbmQgc3VscGhhdGVzIGFyZSBhc3NvY2lhdGVkIHdpdGggcmVkIHdpbmVzLiBUb2dldGhlciwgdGhlc2UgcHJlZGljdG9ycyBleHBsYWluIG11Y2ggb2YgdGhlIHZhcmlhdGlvbiBpbiB0aGUgcHJlZGljdG9yIHZhcmlhYmxlLCB3aW5lIHR5cGUgKHJlc2lkdWFsIGRldmlhbmNlIG9mIG9ubHkgMjg2LjQ5KS4gCgoK